finddalao_sol/src/code/005_dalaotransfer_dropduplicate_solanafm.py

import json


import shutil

import urllib.parse
from base_class import BaseVariableFunction
from base_class import *

old_print = print


def timestamped_print(*args, **kwargs):
    old_print(datetime.datetime.utcnow().replace(
        microsecond=0), *args, **kwargs)


print = timestamped_print
baseclass = BaseVariableFunction(__file__)
print('\n'*5)
print(f"{'{:<6}'.format('ENTER')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------")
utc_timeto = int(time.time())
utc_timefrom = int(utc_timeto - 17*24*3600)

def one_dalaotransfer_dropduplicate(str_dalao_address, success_address_list):
    try:
        # for str_dalao_address in arr_str_dalaoaddress:
        # print(f"enter str_dalao_address= {str_dalao_address}")
        dalao_file = baseclass.dalao_transfer_solanafm_path / \
            f"{str_dalao_address}.json"
        if not dalao_file.exists():
            return
        new_arr_obj_dalao_transaction = []
        new_arr_txhash = []
        try:
            with dalao_file.open(mode='r', encoding='utf-8') as f:
                arr_obj_dalao_transaction = json.load(f)
        except Exception as e:
            print(f"read_json_error str_dalao_address={str_dalao_address}")
            raise
        for obj_dalao_transaction in arr_obj_dalao_transaction:
            txhash = obj_dalao_transaction["transactionHash"]
            timestamp = obj_dalao_transaction["data"][0]["timestamp"]
            if txhash in new_arr_txhash:
                continue
            if timestamp <= utc_timefrom  :
                continue
            new_arr_txhash.append(txhash)
            new_arr_obj_dalao_transaction.append(obj_dalao_transaction)
        with dalao_file.open(mode='w', encoding='utf-8') as f:
            json.dump(new_arr_obj_dalao_transaction, f)
        success_address_list.remove(str_dalao_address)
        print(
            f"str_dalao_address= {str_dalao_address} prelen= {len(arr_obj_dalao_transaction)} aftlen= {len(new_arr_obj_dalao_transaction)} remainder= {len(success_address_list)} \n", end='')
    except Exception as e:
        print(f"error_str_dalao_address={str_dalao_address}")
        raise

def mul_dalaotransfer_dropduplicate():
    success_address_list = arr_str_dalaoaddress.copy()
    for idx in range(0, len(arr_str_dalaoaddress), batchSize):
        thread_list = []
        for cur_idx in range(idx, min(idx+batchSize, len(arr_str_dalaoaddress))):
            thread = threading.Thread(
                target=one_dalaotransfer_dropduplicate,
                args=(arr_str_dalaoaddress[cur_idx],
                      success_address_list)
            )
            thread_list.append(thread)
        for thread in thread_list:
            thread.start()
        for thread in thread_list:
            thread.join()
        time.sleep(1)
    return


arr_str_dalaoaddress = []

df = pd.read_csv(baseclass.dalao_merge_path /
                 "filter_dalao.csv", dtype=object)

arr_str_dalaoaddress = df["dalaoAddress"].tolist()
print('arr_str_dalaoaddress', len(arr_str_dalaoaddress))
batchSize = 170
mul_dalaotransfer_dropduplicate()
print(f"{'{:<6}'.format('END')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------")