viceroy

git clone git://git.codymlewis.com/viceroy.git
Log | Files | Refs | README

commit 43097fd23401060ecff9b55175422a9140591d6f
parent 7456262f042b08756278462685556c844f5fc6cc
Author: Cody Lewis <cody@codymlewis.com>
Date:   Thu,  4 Feb 2021 17:02:21 +1100

Added toggle analysis

Diffstat:
M.gitignore | 2+-
Mmain.py | 58++++++++++++++++++++++++++++++----------------------------
Moptions.json | 4++--
Mplot.py | 3+--
Atoggle_stats.py | 79+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mutils/__init__.py | 11++---------
6 files changed, 115 insertions(+), 42 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -7,5 +7,5 @@ data data/* results.pt bd_results.pt -toggle_record.pkl +toggle_record.csv *.png diff --git a/main.py b/main.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Entry-point to run the program +Entry-point to run the program. Author: Cody Lewis """ @@ -19,10 +19,11 @@ from server.adv_controller import Controller import utils import utils.errors from utils.datasets import load_data +import toggle_stats def index_match(arr): - """Check whether the index within an array is equal to the value""" + """Check whether the index within an array is equal to the value.""" for i, a in enumerate(arr): if i == a: return True @@ -30,7 +31,7 @@ def index_match(arr): def find_shards(num_users, num_classes, classes_per_user): - """Find data class shards according to the parameters""" + """Find data class shards according to the parameters.""" end_halves = [list(range(num_classes)) for _ in range(classes_per_user - 1)] if num_classes / classes_per_user < num_users: @@ -50,7 +51,7 @@ def find_shards(num_users, num_classes, classes_per_user): def run(program_flow, current, run_data): - """Run a part of the program""" + """Run a part of the program.""" try: program_flow[current](run_data) return run_data @@ -79,8 +80,9 @@ def load_test_data(run_data, train, backdoor): backdoor=backdoor ) + def system_setup(run_data): - """Setup the system""" + """Setup the system.""" run_data["options"] = utils.load_options() if run_data["options"].verbosity > 0: print("Options set as:") @@ -115,7 +117,7 @@ def system_setup(run_data): def setup_users(run_data): - """Setup the users/clients for the system""" + """Setup the users/clients for the system.""" run_data["user_classes"] = [ Client if i <= run_data["options"].users * ( 1 - run_data["options"].adversaries['percent_adv']) @@ -142,10 +144,7 @@ def setup_users(run_data): def run_simulations(run_data): - """Run the simulations""" - run_data["sim_confusion_matrices"] = torch.tensor([], dtype=int) - if run_data['options'].adversaries['type'].find('backdoor') >= 0: - run_data["sim_confusion_matrices_bd"] = torch.tensor([], dtype=int) + """Run the simulations.""" for i in range(run_data['sim_number'], run_data["options"].num_sims): print(f"Simulation {i + 1}/{run_data['options'].num_sims}") if not run_data.get('sim_setup'): @@ -183,19 +182,15 @@ def run_simulations(run_data): syb_con=run_data.get('sybil_controller') ) confusion_matrices, cm_bd = run_data['server'].get_conf_matrices() - run_data["sim_confusion_matrices"] = torch.cat( - ( - run_data["sim_confusion_matrices"], - confusion_matrices.unsqueeze(dim=0) - ) - ) + if (scm := run_data.get("sim_confusion_matrices")) is not None: + scm += confusion_matrices + else: + run_data["sim_confusion_matrices"] = confusion_matrices if cm_bd is not None: - run_data["sim_confusion_matrices_bd"] = torch.cat( - ( - run_data["sim_confusion_matrices_bd"], - cm_bd.unsqueeze(dim=0) - ) - ) + if (scmb := run_data.get("sim_confusion_matrices_bd")) is not None: + scmb += cm_bd + else: + run_data["sim_confusion_matrices_bd"] = cm_bd if (con := run_data.get('sybil_controller')) is not None: run_data['controller_toggle'].append(con.toggle_record.copy()) con.setup() @@ -233,6 +228,7 @@ def run_simulations(run_data): print(get_printable_stats(loss, loss_val, stats, stats_val)) return run_data + def get_printable_stats(loss, loss_val, stats, stats_val): return f"Loss: t: {loss}, v: {loss_val}\n" + \ f"Accuracy: t: {stats['accuracy']:%}, " + \ @@ -241,6 +237,7 @@ def get_printable_stats(loss, loss_val, stats, stats_val): f"Attack success rate: t: {stats['attack_success']:%}, " + \ f"v: {stats_val['attack_success']:%}\n" + def gen_conf(run_data, criterion, dl_name): loss, conf_mat = utils.gen_confusion_matrix( run_data["server"].net, @@ -259,22 +256,27 @@ def write_results(run_data): print(f"Writing confusion matrices to {run_data['options'].result_file}...") utils.write_results( run_data["options"].result_file, - run_data["sim_confusion_matrices"] + torch.round( + run_data["sim_confusion_matrices"] / run_data["options"].num_sims + ).long() ) if (scmb := run_data.get('sim_confusion_matrices_bd')) is not None: utils.write_results( f"bd_{run_data['options'].result_file}", - scmb + torch.round( + scmb / run_data["options"].num_sims + ).long() ) if (ct := run_data.get('controller_toggle')) is not None: - tr_fn = 'toggle_record.pkl' + tr_fn = 'toggle_record.csv' print(f"Writing toggle record to {tr_fn}...") max_len = max([len(a) for a in ct]) for a in ct: while len(a) < max_len: - a.append(run_data['options'].server_epochs) - with open(tr_fn, 'wb') as f: - pickle.dump(torch.tensor(ct, dtype=float), f) + a.append(-1) + toggle_stats.write_results( + run_data["options"], torch.tensor(ct), tr_fn + ) if run_data["options"].verbosity > 0: print("Done.") return run_data diff --git a/options.json b/options.json @@ -16,7 +16,7 @@ "params": { "kappa": 2, "importance": false, - "reputation": true + "reputation": false }, "adversaries": { "percent_adv": 0.5, @@ -25,7 +25,7 @@ "to": 0, "scale_up": false, "toggle_times": null, - "delay": 1000, + "delay": 1, "beta": 0.1, "gamma": 0.7, "optimized": true diff --git a/plot.py b/plot.py @@ -10,7 +10,6 @@ Author: Cody Lewis import math from itertools import cycle import re -import pickle import matplotlib.pyplot as plt from matplotlib.patches import Rectangle @@ -96,7 +95,7 @@ def save_plot(options, stats, filter_fn, img_name): title = f"Performance of {options.fit_fun} under {title}" for k in stats.keys(): if filter_fn(k): - ax.plot(epochs, stats[k].mean(dim=0), label=k.replace('_', ' ')) + ax.plot(epochs, stats[k], label=k.replace('_', ' ')) plt.xlabel("Epochs") plt.ylabel("Rate") plt.title(title.title(), fontdict={'fontsize': 7}) diff --git a/toggle_stats.py b/toggle_stats.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Generate stats on the toggles of a simulation + +Author: Cody Lewis +""" + +import pickle + +from pathlib import Path + +import utils + + +def avg(a): + return sum(a) / l if (l := len(a)) > 0 else 0 + + +def get_stats(options, toggle_record): + off_count = 0 + off_to_on_toggles = [] + off_to_on_count = 0 + on_to_off_toggles = [] + on_to_off_count = 0 + for row in toggle_record: + row = row[row != -1] + on = False + last_time = 0 + for cell in row: + on = not on + if on: + off_to_on_toggles.append(cell.item() - last_time) + off_count += off_to_on_toggles[-1] + else: + on_to_off_toggles.append(cell.item() - last_time) + last_time = cell.item() + off_to_on_count += avg(off_to_on_toggles) + on_to_off_count += avg(on_to_off_toggles) + if last_time < options.server_epochs: + if not on: + off_count += options.server_epochs - last_time + on_count = options.num_sims * options.server_epochs - off_count + stats = { + "time_on": on_count, + "time_off": off_count, + "off_to_on": off_to_on_count, + "on_to_off": on_to_off_count + } + for k, v in stats.items(): + stats[k] = v / options.num_sims + return stats + +def write_stats(options, stats, file_name): + if not Path(file_name).is_file(): + with open(file_name, "w") as f: + f.write(f"ds,{','.join(stats.keys())}\n") + with open(file_name, "a") as f: + f.write( + f"{options.dataset}_{options.model_params['architecture']},{','.join([str(v) for v in stats.values()])}\n" + ) + + +def write_results(options, toggle_record, file_name): + stats = get_stats(options, toggle_record) + write_stats(options, stats, file_name) + +if __name__ == '__main__': + import matplotlib.pyplot as plt + import pandas + df = pandas.read_csv('toggle_record.csv') + df.plot(kind='bar') + plt.xticks(range(len(df)), [x[:1] for x in df['ds']]) + plt.xlabel("Dataset and Model") + plt.ylabel("Number of Epochs") + plt.title("Comparison of Attack Timings") + plt.legend(loc=1, fontsize=5, framealpha=0.4) + plt.show() diff --git a/utils/__init__.py b/utils/__init__.py @@ -71,23 +71,16 @@ def gen_conf_stats(confusion_matrix, options): return stats -def gen_experiment_stats(sim_confusion_matrices, options): +def gen_experiment_stats(confusion_matrices, options): """Find the statistics across multiple simulations""" stats = merge_dicts( - [gen_sim_stats(c, options) for c in sim_confusion_matrices] + [gen_conf_stats(c, options) for c in confusion_matrices] ) for k, v in stats.items(): stats[k] = torch.tensor(v) return stats -def gen_sim_stats(confusion_matrices, options): - """Find the stastics of one simulation""" - return merge_dicts( - [gen_conf_stats(c, options) for c in confusion_matrices] - ) - - def merge_dicts(dict_list): """Merge two dictionaries""" merged = {k: [] for k in dict_list[0].keys()}