finddalao_sol/src/code/007_get_configtoken_st_solanafm.py

import json


import shutil

import urllib.parse
from base_class import BaseVariableFunction
from base_class import *

old_print = print


def timestamped_print(*args, **kwargs):
    old_print(datetime.datetime.utcnow().replace(
        microsecond=0), *args, **kwargs)


print = timestamped_print
baseclass = BaseVariableFunction(__file__)
print('\n'*5)
print(f"{'{:<6}'.format('ENTER')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------")
baseclass.makedirpath(baseclass.dalao_configtoken_st_solanafm_path)

Raydium_Authority_V4 = "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1"
WSOL_Token = "So11111111111111111111111111111111111111112"

def checkgdfisswap(gdf, str_dalaoaddress):
    is_raydium =False
    # 保留与raudium 有关的token 的transfer 即只参与了swap的token
    raydium_mask = ((gdf["source"] == Raydium_Authority_V4) | (
        gdf["destination"] == Raydium_Authority_V4))
    filter_dalao_mask = ((gdf["source"] == str_dalaoaddress) | (
        gdf["destination"] == str_dalaoaddress))
    if raydium_mask.any():
        is_raydium =True
        gdf["platform"]="raydium"
 
    arr_raydium_token =None
    arr_dalao_token =None

    if is_raydium:
        arr_raydium_token = gdf[raydium_mask]["token"].drop_duplicates().tolist()
        gdf = gdf[gdf["token"].isin(arr_raydium_token)]
    else:
        arr_dalao_token = gdf[filter_dalao_mask]["token"].drop_duplicates().tolist()
        gdf = gdf[gdf["token"].isin(arr_dalao_token)]
    if not is_raydium:

        dalao_intoken_mask = gdf["destination"] == str_dalaoaddress
        dalao_outtoken_mask = gdf["source"] == str_dalaoaddress
        arr_dalao_intoken = gdf[dalao_intoken_mask]["token"].drop_duplicates(
        ).tolist()
        arr_dalao_outtoken = gdf[dalao_outtoken_mask]["token"].drop_duplicates(
        ).tolist()
        if (len(arr_dalao_intoken) == 0 or len(arr_dalao_outtoken) == 0):
            # dalao 没有token in 或者out 有问题 此sign 不是swap
            return None
        arr_dalao_token = list(set(arr_dalao_intoken) | set(arr_dalao_outtoken))
        if (len(arr_dalao_token) != 2):
            # dalao 的token数目不对 除了 sol token 还有其他的token 不是我想要的sign return
            return None
        if WSOL_Token not in arr_dalao_token:
            # 没有sol 不是我想要的
            return None
        arr_dalao_token.remove(WSOL_Token)
        # token in ,wsol out 或者 token out ,wsol in
        str_token_address = arr_dalao_token[0]
        # if str_token_address in arr_dalao_intoken and 
        if WSOL_Token in arr_dalao_intoken:
            arr_dalao_intoken.remove(WSOL_Token)
        if WSOL_Token in arr_dalao_outtoken:
            arr_dalao_outtoken.remove(WSOL_Token)
        # if str_token_address in arr_dalao_intoken and 
        if len(arr_dalao_intoken) +len(arr_dalao_outtoken)!=1:
            return None
        if str_token_address in arr_dalao_outtoken:
            gdf["swap_type"] = "sell"
        else:
            gdf["swap_type"] = "buy"

        gdf["swap_tokenaddress"] = str_token_address
    else:
        
        raydium_intoken_mask = gdf["destination"] == Raydium_Authority_V4
        raydium_outtoken_mask = gdf["source"] == Raydium_Authority_V4
        arr_raydium_intoken = gdf[raydium_intoken_mask]["token"].drop_duplicates(
        ).tolist()
        arr_raydium_outtoken = gdf[raydium_outtoken_mask]["token"].drop_duplicates(
        ).tolist()
        if (len(arr_raydium_intoken) == 0 or len(arr_raydium_outtoken) == 0):
            # raydium 没有token in 或者out 有问题 此sign 不是swap
            return None
        arr_raydiumtoken = list(set(arr_raydium_intoken) | set(arr_raydium_outtoken))
        if (len(arr_raydiumtoken) != 2):
            # raydium 的token数目不对 除了 sol token 还有其他的token 不是我想要的sign return
            return None
        if WSOL_Token not in arr_raydiumtoken:
            # 没有sol 不是我想要的
            return None
        arr_raydiumtoken.remove(WSOL_Token)
        # token in ,wsol out 或者 token out ,wsol in
        str_token_address = arr_raydiumtoken[0]
        # if str_token_address in arr_raydiumintoken and 
        if WSOL_Token in arr_raydium_intoken:
            arr_raydium_intoken.remove(WSOL_Token)
        if WSOL_Token in arr_raydium_outtoken:
            arr_raydium_outtoken.remove(WSOL_Token)
        # if str_token_address in arr_raydiumintoken and 
        if len(arr_raydium_intoken) +len(arr_raydium_outtoken)!=1:
            return None
        if str_token_address in arr_raydium_outtoken:
            gdf["swap_type"] = "buy"
        else:
            gdf["swap_type"] = "sell"
        gdf["swap_tokenaddress"] = str_token_address
			


    return gdf


def setswaptoken(gdf, str_dalaoaddress):
    arr_dalao_token =gdf["token"].drop_duplicate().tolist()
    # dalao_intoken_mask = gdf["destination"] == str_dalaoaddress
    # dalao_outtoken_mask = gdf["source"] == str_dalaoaddress
    # arr_dalao_intoken = gdf[dalao_intoken_mask]["token"].drop_duplicates(
    # ).tolist()
    # arr_dalao_outtoken = gdf[dalao_outtoken_mask]["token"].drop_duplicates(
    # ).tolist()
    # arr_dalao_token = list(set(arr_dalao_intoken) | set(arr_dalao_outtoken))
    arr_dalao_token.remove(WSOL_Token)
    swapTokenAddress = arr_dalao_token[0]
    gdf["swap_tokenaddress"] = swapTokenAddress
    return gdf


pow_ten_list = [
    '1', '10', '100', '1000',
    '10000', '100000', '1000000',
    '10000000', '100000000', '1000000000',
    '10000000000', '100000000000', '1000000000000',
    '10000000000000', '100000000000000', '1000000000000000',
    '10000000000000000', '100000000000000000', '1000000000000000000',
    '10000000000000000000', '100000000000000000000', '1000000000000000000000',
    '10000000000000000000000', '100000000000000000000000', '1000000000000000000000000'
]


# swap_ethamount	swap_tokenamount


def compute_eth_amount_onerow(row, str_dalaoaddress):
    row = row.copy()
    if (row["token"] != WSOL_Token):
        return row
    amount = decimal.Decimal(row['amount'])
    row['amount'] = '{0:.4f}'.format(
        amount / decimal.Decimal(pow_ten_list[9]))
    platform = row["platform"]
    if platform=="raydium":
        if (row['source'] == Raydium_Authority_V4):
            row['swap_eth_amount'] = '{0:.4f}'.format(decimal.Decimal(
                row['swap_eth_amount']) + decimal.Decimal(row['amount']))
        if (row['destination'] == Raydium_Authority_V4):
            row['swap_eth_amount'] = '{0:.4f}'.format(decimal.Decimal(
                row['swap_eth_amount']) - decimal.Decimal(row['amount']))
    else:
        if (row['source'] == str_dalaoaddress):
            if row['swap_type']=="sell":
                # sell token 只关注 获取eth 不关注其他
                return row
            else:
                row['swap_eth_amount'] = '{0:.4f}'.format(decimal.Decimal(
                    row['swap_eth_amount']) - decimal.Decimal(row['amount']))
            
        if (row['destination'] == str_dalaoaddress):
            row['swap_eth_amount'] = '{0:.4f}'.format(decimal.Decimal(
                row['swap_eth_amount']) + decimal.Decimal(row['amount']))
        

    return row

# def compute_eth_amount_gdf(gdf  ):
#     # 判断此hash 是buy 还是 sell
#     ,,,,
#     return

def compute_eth_amount(df, str_dalaoaddress):
    # 將主流幣全部替換為wbnb，amount 以及 token 都替換，
    # 添加swap_ethamount 列
    # 计算每条hash 的 bnb 的value
     
    df["swap_eth_amount"]="0"
   
    # 待添加
    # df = df.groupby(by=["sign"], group_keys=False).apply(
    # lambda gdf: compute_eth_amount_gdf(gdf))

    df = df.apply(lambda row: compute_eth_amount_onerow(
        row, str_dalaoaddress), axis=1)
    return df


def compute_token_amount_onerow(row, str_dalaoaddress):
    row = row.copy()
    if (row["token"] == WSOL_Token):
        return row
    amount = decimal.Decimal(row['amount'])
    row['amount'] = '{0:.0f}'.format(
        amount / decimal.Decimal(pow_ten_list[6]))
    platform = row["platform"]
    if platform=="raydium":
        if (row['source'] == Raydium_Authority_V4):
            row['swap_token_amount'] = '{0:.0f}'.format(decimal.Decimal(
                row['swap_token_amount']) + decimal.Decimal(row['amount']))
        if (row['destination'] == Raydium_Authority_V4):
            row['swap_token_amount'] = '{0:.0f}'.format(decimal.Decimal(
                row['swap_token_amount']) - decimal.Decimal(row['amount']))
    else:
        if (row['source'] == str_dalaoaddress):
            row['swap_token_amount'] = '{0:.0f}'.format(decimal.Decimal(
            row['swap_token_amount']) - decimal.Decimal(row['amount']))
        if (row['destination'] == str_dalaoaddress):
            row['swap_token_amount'] = '{0:.0f}'.format(decimal.Decimal(
            row['swap_token_amount']) + decimal.Decimal(row['amount']))
    return row


def compute_token_amount(df, str_dalaoaddress):

    df = df.apply(lambda row: compute_token_amount_onerow(
        row, str_dalaoaddress), axis=1)

    return df


def get_mul_configtoken_st():
    success_address_list = arr_str_dalaoaddress.copy()
    for idx in range(0, len(arr_str_dalaoaddress), batchSize):
        thread_list = []
        for cur_idx in range(idx, min(idx+batchSize, len(arr_str_dalaoaddress))):
            thread = threading.Thread(
                target=get_one_configtoken_st,
                args=(arr_str_dalaoaddress[cur_idx],
                      success_address_list)
            )
            thread_list.append(thread)
        for thread in thread_list:
            thread.start()
        for thread in thread_list:
            thread.join()
    return


def get_one_configtoken_st(str_dalaoaddress, success_address_list):
    print(f"enter str_dalaoaddress= {str_dalaoaddress}")
    if not (baseclass.dalao_transaction_st_solanafm_path /
            f"{str_dalaoaddress}.csv").exists():
        return
    df = pd.read_csv(baseclass.dalao_transaction_st_solanafm_path /
                     f"{str_dalaoaddress}.csv", dtype=object)
    if (len(df) == 0):
        return
    # token可能为kong 代表sol相关 用wsol填充
    df["token"] = df["token"].fillna(WSOL_Token)
    prelen = len(df)
    df["platform"]="unknown"
    df["swap_type"]="unknow"
    df["swap_tokenaddress"] = ""
    df = df.groupby(by=["sign"], group_keys=False).apply(
        lambda gdf: checkgdfisswap(gdf, str_dalaoaddress))
    # 屏蔽pump 只考虑raydium 
    # df =df[ df["platform"]=="raydium"].reset_index(drop=True)
    aftlen = len(df)
    if prelen != aftlen:
        print(f"{str_dalaoaddress} pre= {prelen} aft= {aftlen}")
    if aftlen == 0:
        return

    # df = df.groupby(by=["sign"], group_keys=False).apply(
    #     lambda gdf: setswaptoken(gdf, str_dalaoaddress))
    df["swap_eth_amount"] = "0"
    df = compute_eth_amount(df=df, str_dalaoaddress=str_dalaoaddress)

    df["swap_token_amount"] = "0"
    df = compute_token_amount(df=df, str_dalaoaddress=str_dalaoaddress)

    df["timestamp"] = df["timestamp"].astype(int)
    df["token_idx"] = -1
 

    gettokenidx_df = df[["timestamp", "swap_tokenaddress"]].copy()
    gettokenidx_df = gettokenidx_df.sort_values(
        by=["timestamp"], ascending=[True])
    list_tokenaddress = gettokenidx_df["swap_tokenaddress"].drop_duplicates(
    ).tolist()
    for str_tokenaddress_idx in range(0, len(list_tokenaddress)):
        str_tokenaddress = list_tokenaddress[str_tokenaddress_idx]
        df.loc[df["swap_tokenaddress"] == str_tokenaddress,
               "token_idx"] = str_tokenaddress_idx+1

    df["dalaofirsttimestamp"] = df.groupby("token_idx")[
        "timestamp"].transform("min")
    df['dalaofirsttimestamp'] = pd.to_datetime(
        df['dalaofirsttimestamp'], unit='s')

    df = df.sort_values(by=["dalaofirsttimestamp", "token_idx", "timestamp"], ascending=[
        False, True, True]).reset_index(drop=True)
    df.to_csv(baseclass.dalao_configtoken_st_solanafm_path /
              f"configtoken_{str_dalaoaddress}.csv", index=False)
    success_address_list.remove(str_dalaoaddress)
    print(
        f"{str_dalaoaddress} success remainder_len={len(success_address_list)}\n", end='')


batchSize = 170
df = pd.read_csv(baseclass.dalao_merge_path /
                 "filter_dalao.csv", dtype=object)

arr_str_dalaoaddress = df["dalaoAddress"].tolist()
print('arr_str_dalaoaddress', len(arr_str_dalaoaddress))
# arr_str_dalaoaddress=arr_str_dalaoaddress[0:10]
get_mul_configtoken_st()
print(f"{'{:<6}'.format('END')} {baseclass.scriptfilename} ----------------NOTE-----------NOTE---------------")