Заполнение базы plx

#sql_pol.1.4.py
#import numpy
#import talib
import requests
import json
import time
import calendar
import mysql.connector
import sys
from datetime import datetime
#from matplotlib.finance import candlestick2_ohlc
#i#mport matplotlib.animation as animation
#import matplotlib.pyplot as plt
#import matplotlib.ticker as ticker
from datetime import datetime
k=0
NO_DATA = False

def get_date(str_date):
    #https://pythonworld.ru/moduli/modul-time.html
    #print(str_date)
    t1=time.strptime(str_date,'%Y-%m-%d %H:%M:%S')
    #print(t1)
    t2 = calendar.timegm(t1)
    #print(t2)
    #print(datetime.utcfromtimestamp(t2))
    return t2


def get_polo():
    global NO_DATA
    global k
    global st_time
    global end_time
    pair = 'USDT_BTC'
    time.sleep(0.2)
    #chart_data={}
    #manual http://docs.python-requests.org/en/master/user/advanced/#request-and-response-objects
    while True:
        try:
            log('requests.get')
            #response  = requests.get('https://poloniex.com/public?command=returnTradeHistory&currencyPair=%s&start=%s&end=%s' % (pair,st_time,end_time), timeout = None)
            response  = requests.get('https://poloniex.com/public?command=returnTradeHistory&currencyPair=%s&start=%s&end=%s' % (pair,st_time,end_time), timeout = 5)
            log('response.status_code =', response.status_code)
            if response.status_code == 200:
                break
            else:
                log('response.status_code =', response.status_code)
        except Exception as e:
            log(e)
    chart_data = json.loads(response.text)
    k=-1
    if type(chart_data) is list:
        k = len(chart_data)
    #print('st_time = ', st_time)
    #print('end_time = ', end_time)
    #print('k = ',k)
    print(chart_data[-1])
    return chart_data


def update_trade(data):
    PAIR = 'BTC_USD'
    db = mysql.connector.connect(host="localhost", user="u", passwd="secret", db="polo")
    cur = db.cursor()
    #[{'globalTradeID': 355563163, 'tradeID': 20847088, 'date': '2018-03-20 17:31:49', 'type': 'buy', 'rate': '8775.87720002', 'amount': '0.00131316', 'total': '11.52413090'}]   
    #print(data)
    try:
        for item in reversed(data):
            _SQL = """insert ignore into trade
                      (gtrade_id, trade_id, date, type, rate, amount, total)
                      values
                      (%s,%s,%s,%s,%s,%s,%s)"""
            cur.execute(_SQL, (item['globalTradeID'], item['tradeID'], get_date(item['date']), item['type'], item['rate'], item['amount'],item['total']))
            db.commit()
    except Exception as e:
        log(e)
    cur.close()
    db.close()


def verify_trade():
    global st_time
    global end_time
    db = mysql.connector.connect(host="localhost", user="u", passwd="secret", db="polo")
    cur = db.cursor()
    try:
        _SQL = """select count(1) FROM trade
                  WHERE (date >=  %s and date <= %s)"""
        cur.execute(_SQL,(st_time, end_time))
        arr = cur.fetchall()
        #print(arr)
        #print('число записей ', arr[0][0])
        db.commit()
        return int(arr[0][0])
    except Exception as e:
        log(e)
    cur.close()
    db.close()

def update_verify():
    global st_time
    global end_time
    global k
    db = mysql.connector.connect(host="localhost", user="u", passwd="secret", db="polo")
    cur = db.cursor()
    try:
        _SQL = """insert ignore into  verify
                  (st_time, end_time, count)
                  values (%s,%s,%s)"""
        cur.execute(_SQL, (st_time, end_time, k))
        db.commit()
    except Exception as e:
        log(e)
    cur.close()
    db.close()

def get_end_time():
    db = mysql.connector.connect(host="localhost", user="u", passwd="secret", db="polo")
    cur = db.cursor()
    try:
        _SQL = """select end_time from  verify
                  order by end_time desc limit 1"""
        cur.execute(_SQL)
        arr = cur.fetchall()
        #print(arr)
        #print('число записей ', arr[0][0])
        db.commit()
        return int(arr[0][0])
    except Exception as e:
        log(e)
    cur.close()
    db.close()

def count_verify():
    db = mysql.connector.connect(host="localhost", user="u", passwd="secret", db="polo")
    cur = db.cursor()
    try:
        _SQL = """select count(1) FROM verify"""
        cur.execute(_SQL)
        arr = cur.fetchall()
        #print(arr)
        #print('число записей ', arr[0][0])
        db.commit()
        return int(arr[0][0])
    except Exception as e:
        log(e)
    cur.close()
    db.close()

def log(*args):
    #if USE_LOG:
    l = open("./sql_pol.txt", 'a', encoding='utf-8')
    print(datetime.now(), *args, file=l)
    #print(datetime.fromtimestamp(y), *args, file=l) #Локальное время
    #print(datetime.utcfromtimestamp(y), *args, file=l) #UTC
    l.close()
    print(datetime.now(),' ', *args)
    #print(datetime.fromtimestamp(y+3*60*60),' ', *args)
    #print(datetime.utcfromtimestamp(y),' ', *args)


   
#Установим дату и время начала заполнения
# UTC
time_start = (2017, 1, 1, 0, 0, 0, 0, 0, 0)  #Время UTC
timestampstart = calendar.timegm(time_start) # время UTC
#Установим дату и время завершения заполнения
#UTC
if False: #True -заданное время  False текущее
    time_end = (2018, 3, 23, 20, 0, 0, 0, 0, 0) #Время UTC
    timestampend = calendar.timegm(time_end) # время UTC
else:
    timestampend  = time.time()
print('Время завершения выполнения', datetime.utcfromtimestamp(timestampend))


timeplus = 3600
st_time=timestampstart
log('count_verify()= ', count_verify())
if count_verify() > 0:
    log('get_end_time() = ', get_end_time())
    st_time=get_end_time() + 1
    log('Продолжаем работу c ', datetime.utcfromtimestamp(st_time))
log('Начало')
# Бесконечный цикл процесса - основная логика
#update_mysql(get_polo())
#if False:
if True:
    while True:
        #i=1
        #while i<5:
        try:
            if st_time > timestampend:
                log('Завершение заполнения базы', datetime.fromtimestamp(timestampend))
                sys.exit()
            #print('st_time = ', st_time)
            end_time=st_time+timeplus
            #print('end_time = ', end_time)
            i = 0
            while  i < 10 :
                try:
                    Flag = False
                    data=get_polo()
                    if type(data) is list:
                        Flag = True
                        if len(data) > 0:
                            update_trade(data)
                            count=verify_trade()
                            log('Верифицировано!') #Добавляем в sql запись
                            log('с ', datetime.utcfromtimestamp(st_time))
                            log('по ', datetime.utcfromtimestamp(end_time))
                            log('Количество записей ',k)
                            update_verify()
                            k=0
                            break
                        else:
                            log('Список нулевой длины  len(data) :',len(data))
                    else:
                        Flag = False
                        print('type(data) : ', type(data))
                    i = i + 1
                except Exception as e:
                    log(e)
                    time.sleep(1)
            if not Flag :
                log('Аварийное Завершение заполнения базы', datetime.fromtimestamp(st_time))
                log('Попытки получить данные из интернета неудачны')
                sys.exit()
            if k  > 0 :
                update_trade(data)
                count=verify_trade()
            else:
                count = 0
            if count == k :
                log('Верифицировано!') #Добавляем в sql запись
                log('с ', datetime.utcfromtimestamp(st_time))
                log('по ', datetime.utcfromtimestamp(end_time))
                #print('st_time = ', st_time)
                #print('end_time = ', end_time)
                log('Количество записей ',k)
                update_verify()
                k=0
            else:
                log('Аварийное Завершение заполнения базы', datetime.fromtimestamp(st_time))
                log('Не совпадение количества записе!')
                sys.exit()
            st_time=end_time+1 #Предполагаем что правая  граница  на polonix
            time.sleep(1)
            #i=i+1
        except Exception as e:
            log(e)