Initial commit

hap_csv_upload.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import zmq
+import io
+import json
+import csv
+import datetime
+import struct
+
+
+def sec_from_period(period):
+    if period == "M1":
+        return 60
+    elif period == "M5":
+        return 60 * 5
+    elif period == "M15":
+        return 60 * 15
+    elif period == "M30":
+        return 60 * 30
+    elif period == "H1":
+        return 60 * 60
+    elif period == "D":
+        return 86400
+
+def main():
+    parser = argparse.ArgumentParser(description='Finam quote downloader')
+    parser.add_argument('-i', '--input-file', action='store', dest='input_file', help='Input filename', required=True)
+    parser.add_argument('-p', '--timeframe', action='store', dest='timeframe', help='Data timeframe', required=True)
+    parser.add_argument('-o', '--hap', action='store', dest='hap', help='HAP endpoint', required=True)
+    parser.add_argument('-y', '--hap-symbol', action='store', dest='hap_symbol', help='HAP symbol', required=True)
+    parser.add_argument('-d', '--time-delta', action='store', dest='time_delta', help='Time delta (hours)')
+    parser.add_argument('-f', '--force-from', action='store', dest='force_from', help='Force period start')
+    parser.add_argument('-t', '--force-to', action='store', dest='force_to', help='Force period end')
+
+
+    args = parser.parse_args()
+
+    period = args.timeframe
+
+    out_symbol = args.hap_symbol
+
+    ctx = zmq.Context.instance()
+    s = ctx.socket(zmq.REQ)
+    s.connect(args.hap)
+    serialized_bars = io.BytesIO()
+    min_dt = None
+    max_dt = None
+    time_delta = datetime.timedelta(hours=0)
+    if args.time_delta is not None:
+        time_delta = datetime.timedelta(hours=int(args.time_delta))
+        print('Applying delta:', time_delta)
+    line_count = 0
+    with open(args.input_file, 'r') as f:
+        reader = csv.reader(f, delimiter=';')
+        next(reader)
+        for line in reader:
+            line_count += 1
+            date = line[2]
+            time = line[3]
+            open_ = line[4]
+            high = line[5]
+            low = line[6]
+            close = line[7]
+            volume = line[8]
+
+            year = int(date[0:4])
+            month = int(date[4:6])
+            day = int(date[6:8])
+            hour = int(time[0:2])
+            minute = int(time[2:4])
+            second = int(time[4:6])
+
+            dt = datetime.datetime(year, month, day, hour, minute, second, 0, datetime.timezone.utc) - time_delta
+
+            serialized_bars.write(struct.pack("<qddddQ", int(dt.timestamp()), float(open_), float(high), float(low), float(close), int(volume)))
+
+            if min_dt is None:
+                min_dt = dt
+            else:
+                if dt < min_dt:
+                    min_dt = dt
+
+            if max_dt is None:
+                max_dt = dt
+            else:
+                if dt > max_dt:
+                    max_dt = dt
+
+    if args.force_from is not None:
+        min_dt = datetime.datetime.strptime(args.force_from, "%Y%m%d")
+
+    if args.force_to is not None:
+        max_dt = datetime.datetime.strptime(args.force_to, "%Y%m%d")
+
+    rq = {
+        "ticker" : out_symbol,
+        "start_time" : min_dt.strftime("%Y-%m-%dT%H:%M:%S"),
+        "end_time" : max_dt.strftime("%Y-%m-%dT%H:%M:%S"),
+        "timeframe_sec" : sec_from_period(period)
+    }
+
+    print("Read {} lines".format(line_count))
+    raw_data = serialized_bars.getvalue()
+    print("Sending {} bytes".format(len(raw_data)))
+
+    s.send_multipart([bytes(json.dumps(rq), "utf-8"), raw_data])
+    parts = s.recv_multipart()
+    print("Response:", parts)
+
+
+if __name__ == '__main__':
+    main()
+

hap_test.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import zmq
+import io
+import json
+import csv
+import datetime
+import struct
+
+
+def sec_from_period(period):
+    if period == "1min":
+        return 60
+    elif period == "5min":
+        return 60 * 5
+    elif period == "15min":
+        return 60 * 15
+    elif period == "30min":
+        return 60 * 30
+    elif period == "hour":
+        return 60 * 60
+    elif period == "daily":
+        return 86400
+
+def main():
+    parser = argparse.ArgumentParser(description='Finam quote downloader')
+    parser.add_argument('-o', '--hap', action='store', dest='hap', help='HAP endpoint')
+    parser.add_argument('-y', '--hap-symbol', action='store', dest='hap_symbol', help='HAP symbol')
+
+    period = "15min"
+
+    args = parser.parse_args()
+
+    out_symbol = args.hap_symbol
+
+    ctx = zmq.Context.instance()
+    s = ctx.socket(zmq.REQ)
+    s.connect(args.hap)
+    serialized_bars = io.BytesIO()
+    min_dt = None
+    max_dt = None
+    for i in range(0, 10):
+        date = "2020418"
+        time = "10{:02d}00".format(i)
+        open_ = 1
+        high = 2
+        low = 3
+        close = 4
+        volume = 1200
+        dt = datetime.datetime.strptime(date + "_" + time, "%Y%m%d_%H%M%S") - datetime.timedelta(hours=3) # Convert to UTC
+
+        serialized_bars.write(struct.pack("<qddddQ", int(dt.timestamp()), float(open_), float(high), float(low), float(close), int(volume)))
+
+        if min_dt is None:
+            min_dt = dt
+        else:
+            if dt < min_dt:
+                min_dt = dt
+
+        if max_dt is None:
+            max_dt = dt
+        else:
+            if dt > max_dt:
+                max_dt = dt
+
+    rq = {
+        "ticker" : out_symbol,
+        "start_time" : min_dt.strftime("%Y-%m-%dT%H:%M:%S"),
+        "end_time" : max_dt.strftime("%Y-%m-%dT%H:%M:%S"),
+        "timeframe_sec" : sec_from_period(period)
+    }
+
+    s.send_multipart([bytes(json.dumps(rq), "utf-8"), serialized_bars.getvalue()])
+    parts = s.recv_multipart()
+    print(parts)
+
+
+if __name__ == '__main__':
+    main()
+

qhp-download.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import zmq
+import io
+import json
+import csv
+import datetime
+import struct
+
+
+def main():
+    parser = argparse.ArgumentParser(description='QHP client')
+    parser.add_argument('-o', '--output-file', action='store', dest='output_file', help='Output filename', required=True)
+    parser.add_argument('-p', '--timeframe', action='store', dest='timeframe', help='Data timeframe', required=True)
+    parser.add_argument('-q', '--qhp', action='store', dest='qhp', help='QHP endpoint', required=True)
+    parser.add_argument('-y', '--symbol', action='store', dest='symbol', help='Symbol to download', required=True)
+    parser.add_argument('-f', '--from', action='store', dest='from_', help='Starting date', required=True)
+    parser.add_argument('-t', '--to', action='store', dest='to', help='Ending date', required=True)
+
+    args = parser.parse_args()
+
+    period = args.timeframe
+    symbol = args.symbol
+    filename = args.output_file
+
+    ctx = zmq.Context.instance()
+    s = ctx.socket(zmq.REQ)
+    s.connect(args.qhp)
+
+    start_time = datetime.datetime.strptime(args.from_, "%Y%m%d")
+    end_time = datetime.datetime.strptime(args.to, "%Y%m%d")
+
+    rq = {
+        "ticker" : symbol,
+        "from" : start_time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "to" : end_time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "timeframe" : period
+    }
+
+    s.send_multipart([bytes(json.dumps(rq), "utf-8")])
+    parts = s.recv_multipart()
+
+    line_count = 0
+    with open(args.output_file, 'w') as f:
+        writer = csv.writer(f)
+        writer.writerow(['<TICKER>', '<PER>', '<DATE>', '<TIME>', '<OPEN>', '<HIGH>', '<LOW>', '<CLOSE>', '<VOLUME>'])
+        for line in struct.iter_unpack("<qddddQ", parts[0]):
+            line_count += 1
+
+            timestamp = int(line[0])
+            open_ = float(line[1])
+            high = float(line[2])
+            low = float(line[3])
+            close = float(line[4])
+            volume = int(line[5])
+            dt = datetime.datetime.utcfromtimestamp(timestamp)
+
+            writer.writerow([symbol, period, dt.strftime('%Y%m%d'), dt.strftime('%H%M%S'), str(open_), str(high), str(low), str(close), str(volume)])
+
+
+    print("Written {} lines".format(line_count))
+
+if __name__ == '__main__':
+    main()
+