Snippets for Data Science

# drop column
df = df.drop("col1", axis=1)
df = df.drop(["col2", "col3"], axis=1)
df.drop(["col4", "col5"], axis=1, inplace=True)

df = pd.concat([df1, df2, ...])

merged_df = pd.merge(df1, df2, left_on='df1_key', right_on='df2_key', how='inner')

for idx, row in df.iterrows():

def apply_function(df_each_row):
        return row_of_new_col

df["new_col"] = df.apply(apply_function, axis=1)

df.reset_index(drop=True, inplace=True)

plt.figure(figsize=(10,10))
plt.plot([1, 4, 9, 16], [1,2,3,4]) # x: [1,4,9,16] y: [1,2,3,4]
plt.title("title")
plt.xlabel("x")
plt.ylabel("y")
plt.show()

plt.figure(figsize=(10,10))
plt.plot([1, 4, 9, 16], [1,2,3,4], label="train")
plt.plot([2,3,4,5], [1,2,3,4], label="test")
plt.title("title")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()
plt.show()

plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.13),fancybox=True, shadow=False, ncol=5)

plt.figure(figsize=(10,10))
plt.plot([1, 4, 9, 16], [1,2,3,4], label="train", marker=".")
plt.plot([2,3,4,5], [1,2,3,4], label="test", marker=".")
plt.title("title")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()
plt.show()

plt.figure(figsize=(10,10))
plt.plot([1, 4, 9, 16], [1,2,3,4], ".") # x: [1,4,9,16] y: [1,2,3,4]
plt.title("title")
plt.xlabel("x")
plt.ylabel("y")
plt.show()

plt.figure(figsize=(10,10))
plt.plot([1, 4, 9, 16], [1,2,3,4], ",") # x: [1,4,9,16] y: [1,2,3,4]
plt.title("title")
plt.xlabel("x")
plt.ylabel("y")
plt.show()

plt.savefig(file.png, dpi=100, bbox_inches='tight')

import matplotlib

font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 12}

matplotlib.rc('font', **font)

import pickle

row_dict = {"a" : 1}

log_file = os.path.join(".", ".", "file_name") + ".pkl"
if os.path.exists(log_file):
        with open(log_file, 'rb') as file:
        prev_df = pickle.load(file)
        new_df = pd.concat([pd.DataFrame([row_dict]), prev_df]).reset_index(drop=True)
else:
        new_df = pd.DataFrame([row_dict])

with open(log_file, 'wb') as f:
        pickle.dump(new_df, f)

from sklearn.decomposition import PCA

pca = PCA(n_components=2)
pca.fit(df)

df_pca = pca.transform(df)
df1_pca = pca.transform(df1)

plt.figure(figsize=(10,10))
plt.plot(df_pca[:, 0], df_pca[:, 1], label="df")
plt.plot(df1_pca[:, 0], df1_pca[:, 1], label="df1")
plt.title("PCA")
plt.legend()
plt.show()

pd.options.display.max_columns = None
pd.options.display.max_rows = None

#or

pd.set_option('display.max_columns', 100)
pd.set_option('display.max_rows', 100)