import json import shutil import urllib.parse from base_class import BaseVariableFunction from base_class import * old_print = print def timestamped_print(*args, **kwargs): old_print(datetime.datetime.utcnow().replace( microsecond=0), *args, **kwargs) print = timestamped_print baseclass = BaseVariableFunction(__file__) print('\n'*5) print(f"{'{:<6}'.format('ENTER')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------") utc_timeto = int(time.time()) utc_timefrom = int(utc_timeto - 17*24*3600) def one_dalaotransfer_dropduplicate(str_dalao_address, success_address_list): try: # for str_dalao_address in arr_str_dalaoaddress: # print(f"enter str_dalao_address= {str_dalao_address}") dalao_file = baseclass.dalao_transfer_solanafm_path / \ f"{str_dalao_address}.json" if not dalao_file.exists(): return new_arr_obj_dalao_transaction = [] new_arr_txhash = [] try: with dalao_file.open(mode='r', encoding='utf-8') as f: arr_obj_dalao_transaction = json.load(f) except Exception as e: print(f"read_json_error str_dalao_address={str_dalao_address}") raise for obj_dalao_transaction in arr_obj_dalao_transaction: txhash = obj_dalao_transaction["transactionHash"] timestamp = obj_dalao_transaction["data"][0]["timestamp"] if txhash in new_arr_txhash: continue if timestamp <= utc_timefrom : continue new_arr_txhash.append(txhash) new_arr_obj_dalao_transaction.append(obj_dalao_transaction) with dalao_file.open(mode='w', encoding='utf-8') as f: json.dump(new_arr_obj_dalao_transaction, f) success_address_list.remove(str_dalao_address) print( f"str_dalao_address= {str_dalao_address} prelen= {len(arr_obj_dalao_transaction)} aftlen= {len(new_arr_obj_dalao_transaction)} remainder= {len(success_address_list)} \n", end='') except Exception as e: print(f"error_str_dalao_address={str_dalao_address}") raise def mul_dalaotransfer_dropduplicate(): success_address_list = arr_str_dalaoaddress.copy() for idx in range(0, len(arr_str_dalaoaddress), batchSize): thread_list = [] for cur_idx in range(idx, min(idx+batchSize, len(arr_str_dalaoaddress))): thread = threading.Thread( target=one_dalaotransfer_dropduplicate, args=(arr_str_dalaoaddress[cur_idx], success_address_list) ) thread_list.append(thread) for thread in thread_list: thread.start() for thread in thread_list: thread.join() time.sleep(1) return arr_str_dalaoaddress = [] df = pd.read_csv(baseclass.dalao_merge_path / "filter_dalao.csv", dtype=object) arr_str_dalaoaddress = df["dalaoAddress"].tolist() print('arr_str_dalaoaddress', len(arr_str_dalaoaddress)) batchSize = 170 mul_dalaotransfer_dropduplicate() print(f"{'{:<6}'.format('END')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------")