Pull data from ftp and parse

Joel Grunbaum

2022-01-12 611ad7b12fcc6b34229316099ae66bccbcf24ad9

Pull data from ftp and parse

 .gitmodules

@@ -13,3 +13,6 @@
[submodule "rapidxml"]
    path = rapidxml
    url = https://github.com/discord/rapidxml.git
[submodule "ftplibpp"]
    path = ftplibpp
    url = https://github.com/mkulke/ftplibpp.git

 Makefile

@@ -1,6 +1,6 @@
OS=$(shell uname)
CXX=g++
CXXFLAGS=-std=c++20
CXXFLAGS=-std=c++20 -DNOSSL
DEV=1
TEST=1

@@ -12,9 +12,6 @@

ifneq ($(OS),Darwin)
CXXFLAGS += -static
else
CXXFLAGS += -I/usr/local/opt/curl/include
LDFLAGS += -L/usr/local/opt/curl/lib
endif

ifeq ($(TEST),1)
@@ -22,7 +19,8 @@
endif

ws=easywsclient/easywsclient.o
main=${ws} json.o bom.o protocol.o book.o
xml=ftplibpp/ftplib.o
main=${ws} ${xml} json.o bom.o protocol.o book.o

default: test click bot


 bom.cpp

@@ -1,60 +1,55 @@
#include "bom.hpp"

#include "book.hpp"
#include "ftplibpp/ftplib.h"
#include "rapidxml/rapidxml.hpp"
#include <cstddef>
#include <cstdio>
#include <curl/curl.h>
#include <fstream>
#include <iterator>
#include <stdio.h>
#include <string>
#include <unordered_map>
#include <vector>

namespace bom
{
CURL* curl{NULL};
std::string result;

static size_t my_write(void* buffer, std::size_t size, std::size_t nmemb,
                       void* stream)
{
    std::string& text = *static_cast<std::string*>(stream);
    std::size_t totalsize = size * nmemb;
    text.append(static_cast<char*>(buffer), totalsize);
    return totalsize;
}
ftplib* ftp;

void initialise()
{
    curl_global_init(CURL_GLOBAL_DEFAULT);
    curl = curl_easy_init();
    if (curl) {
        curl_easy_setopt(curl, CURLOPT_URL,
                         "ftp://ftp.bom.gov.au/anon/gen/fwo/IDN60920.xml");
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, my_write);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
    }
    ftp = new ftplib();
    ftp->Connect("ftp.bom.gov.au:21");
    ftp->Login("anonymous", "");
}

void destroy() { curl_global_cleanup(); }
void destroy()
{
    ftp->Quit();
    delete ftp;
}

void updateBom(std::unordered_map<std::string, book::Book>& bs)
{
    CURLcode res;
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (CURLE_OK != res) {
        fprintf(stderr, "Curl failed: %d\n", res);
    }
    ftp->Get("bom_data.xml", "/anon/gen/fwo/IDN60920.xml",
             ftplib::transfermode::ascii);
    std::ifstream fs("bom_data.xml");
    std::vector<char> buffer{std::istreambuf_iterator<char>(fs),
                             istreambuf_iterator<char>()};
    buffer.push_back('\0');
    rapidxml::xml_document<> d;
    d.parse<0>(&result[0]);
    rapidxml::xml_node<>* n = d.first_node();
    for (rapidxml::xml_attribute<>* a = n->first_attribute(); a;
         a = a->next_attribute()) {
        std::cout << a->name() << ": " << a->value() << std::endl;
    }
    d.parse<0>(&buffer[0]);
    // Walk stations
    for (rapidxml::xml_node<>* n = d.first_node()->last_node()->first_node(); n;
         n = n->next_sibling()) {
        int bom_id = std::stoi(n->first_attribute()->next_attribute()->value());
        for (auto &i : bs) {
            if (i.second.stationId == bom_id) {
                // Should be apparent temp
                i.second.bomPrice = std::stod(n->first_node()->first_node()->first_node()->first_node()->value());
                std::cout << i.second.product << ", " << i.second.stationId << ", " << i.second.bomPrice << std::endl;
                break;
            }
        }
    }
}
} // namespace bom

 book.cpp

@@ -86,7 +86,7 @@
           double pasFee, double broFee)
    : bidSide{}, askSide{}, productType{productType}, product(product),
      stationId(stationId), unit(unit), expiry(expiry), aggFee(aggFee),
      pasFee(pasFee), broFee(broFee)
      pasFee(pasFee), broFee(broFee), bomPrice(0)
{
}


 book.hpp

@@ -53,6 +53,7 @@
    double aggFee;
    double pasFee;
    double broFee;
    double bomPrice;

    Book();
    Book(ProductTypeEnum productType, std::string product, int stationId,

 ftplibpp

New file
@@ -0,0 +1 @@
Subproject commit 6be689f5cdfa5d4c16c71a314616023e4ce5617a

 protocol.cpp

@@ -109,24 +109,24 @@
    case json::SPREAD_TYPE:
    case json::CALL_TYPE:
    case json::PUT_TYPE:
        announce(bs, dynamic_cast<json::AnnounceMessage*>(message));
        announce(bs, static_cast<json::AnnounceMessage*>(message));
        break;
    case json::SETTLEMENT:
        settle(bs, dynamic_cast<json::SettleMessage*>(message));
        settle(bs, static_cast<json::SettleMessage*>(message));
        break;
    case json::ADDED:
        addedOrder(bs, dynamic_cast<json::AddedMessage*>(message));
        addedOrder(bs, static_cast<json::AddedMessage*>(message));
        break;
    case json::DELETED:
        deletedOrder(bs, dynamic_cast<json::DeletedMessage*>(message));
        deletedOrder(bs, static_cast<json::DeletedMessage*>(message));
        break;
    case json::TRADE:
        tradeOrder(bs, dynamic_cast<json::TradeMessage*>(message));
        tradeOrder(bs, static_cast<json::TradeMessage*>(message));
        break;
    case json::BROKER_REQUEST:
    case json::BROKER_ACK:
    case json::BROKER_CONFIRM:
        broker(bs, dynamic_cast<json::Broker*>(message));
        broker(bs, static_cast<json::Broker*>(message));
        break;
    default:;
    }