From f01aa13db078b63482d8a2ed2fa6a7fd93ee72b7 Mon Sep 17 00:00:00 2001
From: Denis Tereshkin <denis@kasan.ws>
Date: Wed, 22 Apr 2020 15:41:42 +0700
Subject: [PATCH 1/4] hap_csv_upload: parse time delta as seconds

---
 hap_csv_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hap_csv_upload.py b/hap_csv_upload.py
index a1a5658..01a8201 100644
--- a/hap_csv_upload.py
+++ b/hap_csv_upload.py
@@ -49,7 +49,7 @@ def main():
     max_dt = None
     time_delta = datetime.timedelta(hours=0)
     if args.time_delta is not None:
-        time_delta = datetime.timedelta(hours=int(args.time_delta))
+        time_delta = datetime.timedelta(seconds=int(args.time_delta))
         print('Applying delta:', time_delta)
     line_count = 0
     with open(args.input_file, 'r') as f:

From 81bb61f8421426ff553e700634ee684497e09985 Mon Sep 17 00:00:00 2001
From: Denis Tereshkin <denis@kasan.ws>
Date: Wed, 22 Apr 2020 18:21:20 +0700
Subject: [PATCH 2/4] qhp-download: support for multipart data

---
 qhp-download.py | 55 ++++++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/qhp-download.py b/qhp-download.py
index c875d55..2071707 100644
--- a/qhp-download.py
+++ b/qhp-download.py
@@ -113,43 +113,52 @@ def main():
         "timeframe" : period
     }
 
+    print("Sending request:", rq)
     s.send_multipart([bytes(json.dumps(rq), "utf-8")])
-    parts = s.recv_multipart()
+    print("Awaiting response")
+    resp = s.recv()
 
-    print(parts[0])
-    if parts[0] != b'OK':
-       print("Error:", parts[1])
+    print(resp)
+    if resp != b'OK':
+        errmsg = s.recv_string()
+        print("Error:", errmsg)
 
 
     line_count = 0
     with open(args.output_file, 'w', newline='') as f:
         writer = csv.writer(f)
         writer.writerow(['<TICKER>', '<PER>', '<DATE>', '<TIME>', '<OPEN>', '<HIGH>', '<LOW>', '<CLOSE>', '<VOLUME>'])
-        for line in struct.iter_unpack("<qddddQ", parts[1]):
 
-            timestamp = int(line[0])
-            open_ = float(line[1])
-            high = float(line[2])
-            low = float(line[3])
-            close = float(line[4])
-            volume = int(line[5])
-            dt = datetime.datetime.utcfromtimestamp(timestamp) + timedelta
+        while True:
+            if s.getsockopt(zmq.RCVMORE) == 0:
+                break
+            rawdata = s.recv()
+            print("Got chunk: {} bytes".format(len(rawdata)))
+            for line in struct.iter_unpack("<qddddQ", rawdata):
+
+                timestamp = int(line[0])
+                open_ = float(line[1])
+                high = float(line[2])
+                low = float(line[3])
+                close = float(line[4])
+                volume = int(line[5])
+                dt = datetime.datetime.utcfromtimestamp(timestamp) + timedelta
+
+                if agg:
+                    mbar = agg.push_bar(dt, open_, high, low, close, volume)
+                    if mbar is not None:
+                        line_count += 1
+                        writer.writerow([symbol, agg.timeframe, mbar[0].strftime('%Y%m%d'), mbar[0].strftime('%H%M%S'), str(mbar[1]), str(mbar[2]), str(mbar[3]), str(mbar[4]), str(mbar[5])])
+                else:
+                    line_count += 1
+                    writer.writerow([symbol, period, dt.strftime('%Y%m%d'), dt.strftime('%H%M%S'), str(open_), str(high), str(low), str(close), str(volume)])
+
 
             if agg:
-                mbar = agg.push_bar(dt, open_, high, low, close, volume)
+                mbar = agg.get_bar()
                 if mbar is not None:
                     line_count += 1
                     writer.writerow([symbol, agg.timeframe, mbar[0].strftime('%Y%m%d'), mbar[0].strftime('%H%M%S'), str(mbar[1]), str(mbar[2]), str(mbar[3]), str(mbar[4]), str(mbar[5])])
-            else:
-                line_count += 1
-                writer.writerow([symbol, period, dt.strftime('%Y%m%d'), dt.strftime('%H%M%S'), str(open_), str(high), str(low), str(close), str(volume)])
-
-
-        if agg:
-            mbar = agg.get_bar()
-            if mbar is not None:
-                line_count += 1
-                writer.writerow([symbol, agg.timeframe, mbar[0].strftime('%Y%m%d'), mbar[0].strftime('%H%M%S'), str(mbar[1]), str(mbar[2]), str(mbar[3]), str(mbar[4]), str(mbar[5])])
         
 
     print("Written {} lines".format(line_count))

From 86b639c8c969c0f17e5b6269aa840b3708a8c596 Mon Sep 17 00:00:00 2001
From: Denis Tereshkin <denis@kasan.ws>
Date: Mon, 3 May 2021 09:53:29 +0700
Subject: [PATCH 3/4] Bar aggregation fix

---
 qhp-download.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/qhp-download.py b/qhp-download.py
index 2071707..d4373b0 100644
--- a/qhp-download.py
+++ b/qhp-download.py
@@ -44,7 +44,8 @@ class BarAggregator:
             b_low = self.low
             b_close = self.close
             b_volume = self.volume
-            b_timestamp = self.timestamp
+            if self.current_bar_number is not None:
+                b_timestamp = datetime.datetime.fromtimestamp(self.current_bar_number * self.timeframe)
 
             self.open_ = open_
             self.high = high
@@ -70,7 +71,7 @@ class BarAggregator:
         b_low = self.low
         b_close = self.close
         b_volume = self.volume
-        b_timestamp = self.timestamp
+        b_timestamp = datetime.datetime.fromtimestamp(self.timeframe * ( self.timestamp.timestamp() // self.timeframe))
 
         return (b_timestamp, b_open, b_high, b_low, b_close, b_volume)
 
@@ -122,6 +123,7 @@ def main():
     if resp != b'OK':
         errmsg = s.recv_string()
         print("Error:", errmsg)
+        sys.exit(1)
 
 
     line_count = 0

From 320531a4f7b69cb3d6af317d17b942de5fbeb2e9 Mon Sep 17 00:00:00 2001
From: Denis Tereshkin <denis@kasan.ws>
Date: Mon, 3 May 2021 09:55:09 +0700
Subject: [PATCH 4/4] qhp-download-list tool

---
 qhp-download-list.py | 48 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 qhp-download-list.py

diff --git a/qhp-download-list.py b/qhp-download-list.py
new file mode 100644
index 0000000..8ff2e5d
--- /dev/null
+++ b/qhp-download-list.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import zmq
+import io
+import json
+import csv
+import datetime
+import struct
+
+def main():
+    parser = argparse.ArgumentParser(description='QHP client')
+    parser.add_argument('-q', '--qhp', action='store', dest='qhp', help='QHP endpoint', required=True)
+
+    args = parser.parse_args()
+
+    ctx = zmq.Context.instance()
+    s = ctx.socket(zmq.REQ)
+    s.connect(args.qhp)
+
+    rq = {
+        "get_sec_list" : True,
+    }
+
+    s.send_multipart([bytes(json.dumps(rq), "utf-8")])
+    resp = s.recv()
+
+    if resp != b'OK':
+        errmsg = s.recv_string()
+        print("Error:", errmsg)
+        sys.exit(1)
+
+
+    rawdata = b''
+    while True:
+        if s.getsockopt(zmq.RCVMORE) == 0:
+            break
+        rawdata += s.recv()
+
+    s = rawdata.decode('utf-8')
+    tickers = s.split(',')
+    for ticker in tickers:
+        print(ticker)
+
+if __name__ == '__main__':
+    main()
+