diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/results/2vs4_tlratio.png b/benchmarks/results/2vs4_tlratio.png index 274ede9..b76f296 100644 --- a/benchmarks/results/2vs4_tlratio.png +++ b/benchmarks/results/2vs4_tlratio.png Binary files differ diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/results/2vs4_tlratio.png b/benchmarks/results/2vs4_tlratio.png index 274ede9..b76f296 100644 --- a/benchmarks/results/2vs4_tlratio.png +++ b/benchmarks/results/2vs4_tlratio.png Binary files differ diff --git a/demo/progressive_gen_demo.py b/demo/progressive_gen_demo.py new file mode 100644 index 0000000..44835df --- /dev/null +++ b/demo/progressive_gen_demo.py @@ -0,0 +1,54 @@ +import expyriment +from expyriment import design, control, stimuli, misc +from generators.nb_gm_004 import SequenceGenerator + +import benchmarks.common as common +num_of_blocks = 2 +trials_per_block = 8 +n = 2 +lures = 1 +targets = 2 + +control.defaults.window_mode = True +generators = [SequenceGenerator(common.choices, n) for b in range(0, num_of_blocks)] + +exp = design.Experiment("Progressive N-Back Task") +control.initialize(exp) +exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] + +fixation = stimuli.FixCross() +fixation.preload() + +# left and right arrow keys for responses +response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] + +expyriment.control.start(exp) + + +# initialize three blocks +for b in range(0, num_of_blocks): + block = design.Block() + block.set_factor("block", b) + block.set_factor("trials", trials_per_block) + exp.add_block(block) + +for block in exp.blocks: + trials = block.get_factor("trials") + block_index = block.get_factor("block") + # TODO recalculate N based on performance + generators[block_index].reset(trials_per_block, targets, lures, n=n) + for i in range(0, trials): + trial = design.Trial() + item = generators[block_index].next_trial() + trial.set_factor("stimulus", item) + stim = stimuli.TextLine(item, text_size=200) + exp.clock.wait(1000 - fixation.present() - stim.preload()) + stim.present() + + # Wait for response, remove duration param for forced-choice. + pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) + exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) + exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) + +expyriment.control.end("Goodbye!", goodbye_delay=1000) + diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/results/2vs4_tlratio.png b/benchmarks/results/2vs4_tlratio.png index 274ede9..b76f296 100644 --- a/benchmarks/results/2vs4_tlratio.png +++ b/benchmarks/results/2vs4_tlratio.png Binary files differ diff --git a/demo/progressive_gen_demo.py b/demo/progressive_gen_demo.py new file mode 100644 index 0000000..44835df --- /dev/null +++ b/demo/progressive_gen_demo.py @@ -0,0 +1,54 @@ +import expyriment +from expyriment import design, control, stimuli, misc +from generators.nb_gm_004 import SequenceGenerator + +import benchmarks.common as common +num_of_blocks = 2 +trials_per_block = 8 +n = 2 +lures = 1 +targets = 2 + +control.defaults.window_mode = True +generators = [SequenceGenerator(common.choices, n) for b in range(0, num_of_blocks)] + +exp = design.Experiment("Progressive N-Back Task") +control.initialize(exp) +exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] + +fixation = stimuli.FixCross() +fixation.preload() + +# left and right arrow keys for responses +response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] + +expyriment.control.start(exp) + + +# initialize three blocks +for b in range(0, num_of_blocks): + block = design.Block() + block.set_factor("block", b) + block.set_factor("trials", trials_per_block) + exp.add_block(block) + +for block in exp.blocks: + trials = block.get_factor("trials") + block_index = block.get_factor("block") + # TODO recalculate N based on performance + generators[block_index].reset(trials_per_block, targets, lures, n=n) + for i in range(0, trials): + trial = design.Trial() + item = generators[block_index].next_trial() + trial.set_factor("stimulus", item) + stim = stimuli.TextLine(item, text_size=200) + exp.clock.wait(1000 - fixation.present() - stim.preload()) + stim.present() + + # Wait for response, remove duration param for forced-choice. + pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) + exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) + exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) + +expyriment.control.end("Goodbye!", goodbye_delay=1000) + diff --git a/demo/progressive_random.py b/demo/progressive_random.py deleted file mode 100644 index 50babe3..0000000 --- a/demo/progressive_random.py +++ /dev/null @@ -1,50 +0,0 @@ -import expyriment -from expyriment import design, control, stimuli, misc -from generators import progressive_random - -num_of_blocks = 2 -trials_per_block = 16 -n = 2 -choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] - -control.defaults.window_mode = True -generators = [progressive_random.SequenceGenerator(choices, trials=trials_per_block, n=n) for b in range(0, num_of_blocks)] - -exp = design.Experiment("Progressive N-Back Task") -control.initialize(exp) -exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] - -fixation = stimuli.FixCross() -fixation.preload() - -# left and right arrow keys for responses -response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] - -expyriment.control.start(exp) - - -# initialize three blocks -for b in range(0, num_of_blocks): - block = design.Block() - block.set_factor("block", b) - block.set_factor("trials", trials_per_block) - exp.add_block(block) - -for block in exp.blocks: - trials = block.get_factor("trials") - block_index = block.get_factor("block") - for i in range(0, trials): - trial = design.Trial() - item = generators[block_index].next_trial() - trial.set_factor("stimulus", item) - stim = stimuli.TextLine(item, text_size=200) - exp.clock.wait(1000 - fixation.present() - stim.preload()) - stim.present() - - # Wait for response, remove duration param for forced-choice. - pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) - exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) - exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) - -expyriment.control.end("Goodbye!", goodbye_delay=1000) - diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/results/2vs4_tlratio.png b/benchmarks/results/2vs4_tlratio.png index 274ede9..b76f296 100644 --- a/benchmarks/results/2vs4_tlratio.png +++ b/benchmarks/results/2vs4_tlratio.png Binary files differ diff --git a/demo/progressive_gen_demo.py b/demo/progressive_gen_demo.py new file mode 100644 index 0000000..44835df --- /dev/null +++ b/demo/progressive_gen_demo.py @@ -0,0 +1,54 @@ +import expyriment +from expyriment import design, control, stimuli, misc +from generators.nb_gm_004 import SequenceGenerator + +import benchmarks.common as common +num_of_blocks = 2 +trials_per_block = 8 +n = 2 +lures = 1 +targets = 2 + +control.defaults.window_mode = True +generators = [SequenceGenerator(common.choices, n) for b in range(0, num_of_blocks)] + +exp = design.Experiment("Progressive N-Back Task") +control.initialize(exp) +exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] + +fixation = stimuli.FixCross() +fixation.preload() + +# left and right arrow keys for responses +response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] + +expyriment.control.start(exp) + + +# initialize three blocks +for b in range(0, num_of_blocks): + block = design.Block() + block.set_factor("block", b) + block.set_factor("trials", trials_per_block) + exp.add_block(block) + +for block in exp.blocks: + trials = block.get_factor("trials") + block_index = block.get_factor("block") + # TODO recalculate N based on performance + generators[block_index].reset(trials_per_block, targets, lures, n=n) + for i in range(0, trials): + trial = design.Trial() + item = generators[block_index].next_trial() + trial.set_factor("stimulus", item) + stim = stimuli.TextLine(item, text_size=200) + exp.clock.wait(1000 - fixation.present() - stim.preload()) + stim.present() + + # Wait for response, remove duration param for forced-choice. + pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) + exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) + exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) + +expyriment.control.end("Goodbye!", goodbye_delay=1000) + diff --git a/demo/progressive_random.py b/demo/progressive_random.py deleted file mode 100644 index 50babe3..0000000 --- a/demo/progressive_random.py +++ /dev/null @@ -1,50 +0,0 @@ -import expyriment -from expyriment import design, control, stimuli, misc -from generators import progressive_random - -num_of_blocks = 2 -trials_per_block = 16 -n = 2 -choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] - -control.defaults.window_mode = True -generators = [progressive_random.SequenceGenerator(choices, trials=trials_per_block, n=n) for b in range(0, num_of_blocks)] - -exp = design.Experiment("Progressive N-Back Task") -control.initialize(exp) -exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] - -fixation = stimuli.FixCross() -fixation.preload() - -# left and right arrow keys for responses -response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] - -expyriment.control.start(exp) - - -# initialize three blocks -for b in range(0, num_of_blocks): - block = design.Block() - block.set_factor("block", b) - block.set_factor("trials", trials_per_block) - exp.add_block(block) - -for block in exp.blocks: - trials = block.get_factor("trials") - block_index = block.get_factor("block") - for i in range(0, trials): - trial = design.Trial() - item = generators[block_index].next_trial() - trial.set_factor("stimulus", item) - stim = stimuli.TextLine(item, text_size=200) - exp.clock.wait(1000 - fixation.present() - stim.preload()) - stim.present() - - # Wait for response, remove duration param for forced-choice. - pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) - exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) - exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) - -expyriment.control.end("Goodbye!", goodbye_delay=1000) - diff --git a/generators/nb_gm_004.py b/generators/nb_gm_004.py index 56678a3..87dd447 100644 --- a/generators/nb_gm_004.py +++ b/generators/nb_gm_004.py @@ -25,16 +25,28 @@ self.lures_norm = None self.targets_norm = None - def reset(self, trials, targets, lures): + self.current_trial = None + + def reset(self, trials, targets, lures, **kwargs): self.trials = trials self.targets_ratio = targets / trials self.lures_ratio = lures / trials + self.n = kwargs.get("n", self.n) self.targets_norm = scipy.stats.norm(self.targets_ratio, 1.0) self.lures_norm = scipy.stats.norm(self.lures_ratio, 1.0) self.skewness_norm = scipy.stats.norm(0, 0.5) self.seq = [] + self.current_trial = 0 + + def next_trial(self): + while self.current_trial >= len(self.seq): + chunk_size = 3 if len(self.seq) + 3 <= self.trials else self.trials-len(self.seq) + self.seq += self.best_chunk(chunk_size) + self.current_trial += 1 + print(self.current_trial) + return self.seq[self.current_trial-1] if self.current_trial<=self.trials else None def generate(self, trials, targets, lures): self.reset(trials, targets, lures) @@ -66,7 +78,7 @@ return [best_choice] def cost(self, seq): - targets, lures = common.count_targets_and_lures(seq, n) + targets, lures = common.count_targets_and_lures(seq, self.n) #print(seq, targets, lures) c1, c2, c3 = self.skewness_cost(seq), self.targets_cost(targets/len(seq)), self.lures_cost(lures/len(seq)) return c1+c2+c3 diff --git a/benchmarks/benchmark_diagrams.py b/benchmarks/benchmark_diagrams.py new file mode 100644 index 0000000..8fa448e --- /dev/null +++ b/benchmarks/benchmark_diagrams.py @@ -0,0 +1,159 @@ +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt + +import benchmarks.common as common + + +def tlratio(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2,ignore_index=True) + stats = [] + for trials in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == trials) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == trials) + targets2 = data[alg2].targets if alg2.shape[0] > 0 else np.nan + targets4 = data[alg4].targets if alg4.shape[0] > 0 else np.nan + lures2 = data[alg2].lures if alg2.shape[0] > 0 else np.nan + lures4 = data[alg4].lures if alg4.shape[0] > 0 else np.nan + stats.append([trials, (targets2/lures2).mean(), (targets4/lures4).mean()]) + stats = pd.DataFrame(stats, columns=['trials', 'tl2', 'tl4']) #.dropna(subset=['skewness']) + + plt.style.use('ggplot') + + plt.ylim(bottom=0, top=30) + plt.scatter(stats.trials, stats.tl2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl2[idx], 1)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.tl4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.tl4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.tl4[idx], 2)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('T:L Ratio') + plt.ylabel('T:L Ratio') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_tlratio.png', bbox_inches='tight') + plt.show() + + +def time_per_trial(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for t in range(common.trials_range[1]): + alg2 = (data.alg == 'nb_gm_002') & (data.trials == t) + alg4 = (data.alg == 'nb_gm_004') & (data.trials == t) + time2 = data[alg2].time.mean() * 1000 if alg2.shape[0] > 0 else np.nan + time4 = data[alg4].time.mean() * 1000 if alg4.shape[0] > 0 else np.nan + stats.append([t, time2, time4]) + stats = pd.DataFrame(stats, columns=['trials', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.trials, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time2) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time2[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='blue') + + plt.scatter(stats.trials, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + idx = np.isfinite(stats.trials) & np.isfinite(stats.time4) + p = np.poly1d(np.polyfit(stats.trials[idx], stats.time4[idx], 3)) + plt.plot(stats.trials, p(stats.trials), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('Trials') + + plt.legend() + + plt.savefig(f'benchmarks/results/2v4_time_trials.png', bbox_inches='tight') + plt.show() + +def time_profiles(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 8): + alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + time2 = data[alg2].time.mean() * 1000 + time4 = data[alg4].time.mean() * 1000 + stats.append([ni, time2, time4]) + stats = pd.DataFrame(stats, columns=['n', 'time2', 'time4']) #.dropna(subset=['skewness']) + + print(stats) + plt.style.use('ggplot') + plt.scatter(stats.n, stats.time2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.time2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + plt.scatter(stats.n, stats.time4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.time4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Time Profile') + plt.ylabel('Time (ms)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_time_n.png', bbox_inches='tight') + plt.show() + +def skewness(): + data = pd.read_csv('benchmarks/results/nb_gm_004_profile.csv') + data2 = pd.read_csv('benchmarks/results/nb_gm_002_profile.csv') + + data = data.append(data2) + stats = [] + for ni in range(2, 9): + all_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) + skewed_alg2 = (data.alg == 'nb_gm_002') & (data.n == ni) & data.skewed + all_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) + skewed_alg4 = (data.alg == 'nb_gm_004') & (data.n == ni) & data.skewed + num_of_sequences2 = data[all_alg2].shape[0] + skewed_sequences2 = data[skewed_alg2].shape[0] + num_of_sequences4 = data[all_alg4].shape[0] + skewed_sequences4 = data[skewed_alg4].shape[0] + skewness2 = skewed_sequences2 * 100.0 / num_of_sequences2 if (num_of_sequences2>0) else 0 + skewness4 = skewed_sequences4 * 100.0 / num_of_sequences4 if (num_of_sequences4>0) else 0 + stats.append([ni, skewness2, skewness4]) + stats = pd.DataFrame(stats, columns=['n', 'skewness2', 'skewness4']) #.dropna(subset=['skewness']) + + + plt.style.use('ggplot') + plt.scatter(stats.n, stats.skewness2, color='blue', alpha=0.2, label='nb_gm_002') + p = np.poly1d(np.polyfit(stats.n, stats.skewness2, 3)) + plt.plot(stats.n, p(stats.n), color='blue') + + + plt.scatter(stats.n, stats.skewness4, color='red', alpha=0.2, label='nb_gm_004') + p = np.poly1d(np.polyfit(stats.n, stats.skewness4, 3)) + plt.plot(stats.n, p(stats.n), color='red') + + # plt.title('Skewness') + plt.ylabel('Skewed Blocks (%)') + plt.xlabel('N') + + plt.legend() + + plt.savefig(f'benchmarks/results/2vs4_skewness.png', bbox_inches='tight') + plt.show() + +if __name__ == "__main__": + tlratio() + skewness() + time_profiles() + time_per_trial() diff --git a/benchmarks/results/2v4_time_trials.png b/benchmarks/results/2v4_time_trials.png index e28fdcb..9be91b4 100644 --- a/benchmarks/results/2v4_time_trials.png +++ b/benchmarks/results/2v4_time_trials.png Binary files differ diff --git a/benchmarks/results/2vs4_skewness.png b/benchmarks/results/2vs4_skewness.png index 93743c5..a488e1f 100644 --- a/benchmarks/results/2vs4_skewness.png +++ b/benchmarks/results/2vs4_skewness.png Binary files differ diff --git a/benchmarks/results/2vs4_time_n.png b/benchmarks/results/2vs4_time_n.png index 93ffae9..d022ed0 100644 --- a/benchmarks/results/2vs4_time_n.png +++ b/benchmarks/results/2vs4_time_n.png Binary files differ diff --git a/benchmarks/results/2vs4_tlratio.png b/benchmarks/results/2vs4_tlratio.png index 274ede9..b76f296 100644 --- a/benchmarks/results/2vs4_tlratio.png +++ b/benchmarks/results/2vs4_tlratio.png Binary files differ diff --git a/demo/progressive_gen_demo.py b/demo/progressive_gen_demo.py new file mode 100644 index 0000000..44835df --- /dev/null +++ b/demo/progressive_gen_demo.py @@ -0,0 +1,54 @@ +import expyriment +from expyriment import design, control, stimuli, misc +from generators.nb_gm_004 import SequenceGenerator + +import benchmarks.common as common +num_of_blocks = 2 +trials_per_block = 8 +n = 2 +lures = 1 +targets = 2 + +control.defaults.window_mode = True +generators = [SequenceGenerator(common.choices, n) for b in range(0, num_of_blocks)] + +exp = design.Experiment("Progressive N-Back Task") +control.initialize(exp) +exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] + +fixation = stimuli.FixCross() +fixation.preload() + +# left and right arrow keys for responses +response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] + +expyriment.control.start(exp) + + +# initialize three blocks +for b in range(0, num_of_blocks): + block = design.Block() + block.set_factor("block", b) + block.set_factor("trials", trials_per_block) + exp.add_block(block) + +for block in exp.blocks: + trials = block.get_factor("trials") + block_index = block.get_factor("block") + # TODO recalculate N based on performance + generators[block_index].reset(trials_per_block, targets, lures, n=n) + for i in range(0, trials): + trial = design.Trial() + item = generators[block_index].next_trial() + trial.set_factor("stimulus", item) + stim = stimuli.TextLine(item, text_size=200) + exp.clock.wait(1000 - fixation.present() - stim.preload()) + stim.present() + + # Wait for response, remove duration param for forced-choice. + pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) + exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) + exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) + +expyriment.control.end("Goodbye!", goodbye_delay=1000) + diff --git a/demo/progressive_random.py b/demo/progressive_random.py deleted file mode 100644 index 50babe3..0000000 --- a/demo/progressive_random.py +++ /dev/null @@ -1,50 +0,0 @@ -import expyriment -from expyriment import design, control, stimuli, misc -from generators import progressive_random - -num_of_blocks = 2 -trials_per_block = 16 -n = 2 -choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] - -control.defaults.window_mode = True -generators = [progressive_random.SequenceGenerator(choices, trials=trials_per_block, n=n) for b in range(0, num_of_blocks)] - -exp = design.Experiment("Progressive N-Back Task") -control.initialize(exp) -exp.data_variable_names = ["block", "stimulus", "pressed_key", "rt"] - -fixation = stimuli.FixCross() -fixation.preload() - -# left and right arrow keys for responses -response_keys = [misc.constants.K_LEFT, misc.constants.K_RIGHT] - -expyriment.control.start(exp) - - -# initialize three blocks -for b in range(0, num_of_blocks): - block = design.Block() - block.set_factor("block", b) - block.set_factor("trials", trials_per_block) - exp.add_block(block) - -for block in exp.blocks: - trials = block.get_factor("trials") - block_index = block.get_factor("block") - for i in range(0, trials): - trial = design.Trial() - item = generators[block_index].next_trial() - trial.set_factor("stimulus", item) - stim = stimuli.TextLine(item, text_size=200) - exp.clock.wait(1000 - fixation.present() - stim.preload()) - stim.present() - - # Wait for response, remove duration param for forced-choice. - pressed_key, rt = exp.keyboard.wait(response_keys, duration=2000) - exp.data.add([block.get_factor("block"), trial.get_factor("stimulus"), pressed_key, rt]) - exp.clock.wait(1000 - expyriment.stimuli.BlankScreen().present() - stim.unload()) - -expyriment.control.end("Goodbye!", goodbye_delay=1000) - diff --git a/generators/nb_gm_004.py b/generators/nb_gm_004.py index 56678a3..87dd447 100644 --- a/generators/nb_gm_004.py +++ b/generators/nb_gm_004.py @@ -25,16 +25,28 @@ self.lures_norm = None self.targets_norm = None - def reset(self, trials, targets, lures): + self.current_trial = None + + def reset(self, trials, targets, lures, **kwargs): self.trials = trials self.targets_ratio = targets / trials self.lures_ratio = lures / trials + self.n = kwargs.get("n", self.n) self.targets_norm = scipy.stats.norm(self.targets_ratio, 1.0) self.lures_norm = scipy.stats.norm(self.lures_ratio, 1.0) self.skewness_norm = scipy.stats.norm(0, 0.5) self.seq = [] + self.current_trial = 0 + + def next_trial(self): + while self.current_trial >= len(self.seq): + chunk_size = 3 if len(self.seq) + 3 <= self.trials else self.trials-len(self.seq) + self.seq += self.best_chunk(chunk_size) + self.current_trial += 1 + print(self.current_trial) + return self.seq[self.current_trial-1] if self.current_trial<=self.trials else None def generate(self, trials, targets, lures): self.reset(trials, targets, lures) @@ -66,7 +78,7 @@ return [best_choice] def cost(self, seq): - targets, lures = common.count_targets_and_lures(seq, n) + targets, lures = common.count_targets_and_lures(seq, self.n) #print(seq, targets, lures) c1, c2, c3 = self.skewness_cost(seq), self.targets_cost(targets/len(seq)), self.lures_cost(lures/len(seq)) return c1+c2+c3 diff --git a/generators/progressive_random.py b/generators/progressive_random.py index 441656f..6496761 100644 --- a/generators/progressive_random.py +++ b/generators/progressive_random.py @@ -39,7 +39,7 @@ min_cost, best_choice = sys.float_info.max, None random.shuffle(choices) # to avoid ordering effect for choice in choices: - cost = self.cost(seq + choice) + cost = self.cost(seq + [choice]) if cost < min_cost: min_cost = cost best_choice = choice