Commit 027ff06e authored by cif2cif's avatar cif2cif
Browse files

Unificados nombres de variables y colocados al ppio del fichero; actualizado el README

parent 2ee06f88
......@@ -7,8 +7,10 @@ $ pip install -r requirements.txt
## Configuration
The file `conv1d_autoencoder.py' should be configured as follows:
* variable `normal_data_path`: path to the .pkl file that contains data without attacks, that is to say, a normal behaviour of the houses.
* variable `attack_data_path`: path to the .pkl file that contains the data that is wanted to be analyzed in order to detect attacks.
* variable `NORMAL_DATA_PATH`: path to the .pkl file that contains data without attacks, that is to say, a normal behaviour of the houses.
* variable `ATTACK_DATA_PATH`: path to the .pkl file that contains the data that is wanted to be analyzed in order to detect attacks.
* variable `DATA_DIR-PATH`: path to the folder that contains the folders `NORMAL_DATA_PATH` folder and `ATTACK_DATA_PATH` folder
* variable `MODEL_DIR_PATH`: path to the models that are generated during the training
* variable `DO_TRAINING`: it should be set to True in order to train the autoencoder
## Usage
......
......@@ -16,16 +16,17 @@ from keras_anomaly_detection.library.convolutional import Conv1DAutoEncoder
DO_TRAINING = False
DATA_DIR_PATH = './data'
MODEL_DIR_PATH = './models'
NORMAL_DATA_PATH = '/normal/houses_concatenated.pkl'
# ATTACK_DATA_PATH = '/anomaly_20/labels_df/house_0.pkl'
ATTACK_DATA_PATH = '/anomaly_20/labels_df/houses_concatenated.pkl'
OUTPUT_FILE = 'predicted_labels.csv'
def main():
data_dir_path = './data'
model_dir_path = './models'
normal_data_path = '/normal/houses_concatenated.pkl'
# attack_data_path = '/anomaly_20/labels_df/house_0.pkl'
attack_data_path = '/anomaly_20/labels_df/houses_concatenated.pkl'
#Read Normal Data (No Attacks)
houses = pd.read_pickle(data_dir_path + normal_data_path)
houses = pd.read_pickle(DATA_DIR_PATH + NORMAL_DATA_PATH)
print(houses.head())
houses = houses.drop("attacked", axis=1)
......@@ -37,12 +38,12 @@ def main():
ae = Conv1DAutoEncoder()
# fit the data and save model into model_dir_path
# fit the data and save model into MODEL_DIR_PATH
if DO_TRAINING:
ae.fit(train, model_dir_path=model_dir_path)
ae.fit(train, MODEL_DIR_PATH=MODEL_DIR_PATH)
# load back the model saved in model_dir_path detect anomaly
ae.load_model(model_dir_path)
# load back the model saved in MODEL_DIR_PATH detect anomaly
ae.load_model(MODEL_DIR_PATH)
# generate a graphic of the autoencoder network
......@@ -67,14 +68,14 @@ def main():
final_fs = []
df_house_normal = pd.read_pickle(data_dir_path + normal_data_path)
df_house_normal = pd.read_pickle(DATA_DIR_PATH + NORMAL_DATA_PATH)
labels_normal = df_house_normal["attacked"]
df_house_normal = df_house_normal.drop("attacked", axis=1)
scaled_house_normal = scaler.transform(df_house_normal)
# Read Attack Data
df_house_anomaly = pd.read_pickle(data_dir_path + attack_data_path)
df_house_anomaly = pd.read_pickle(DATA_DIR_PATH + ATTACK_DATA_PATH)
labels_anomaly = df_house_anomaly["attacked"]
df_house_anomaly = df_house_anomaly.drop("attacked", axis=1)
......@@ -100,10 +101,15 @@ def main():
plt.plot(normalized_rsme, marker='o', linestyle='', ms=3.5)
plt.show()
print("POSTPLOT")
final_labels = window_error(normalized_rsme, labels_anomaly)
outfile = open('predicted_labels.csv','w')
print("FINAL-LABELS SET")
outfile = open(OUTPUT_FILE,'w')
print("output predictedcsv")
out = csv.writer(outfile)
out.writerows(map(lambda x: [x], final_labels))
outfile.close()
......@@ -114,6 +120,7 @@ def main():
def get_th_opt(normalized_rsme, labels_anomaly, flag):
print("entro get_th_opt")
j=2
predicted_labels = {}
labels = labels_anomaly
......@@ -144,7 +151,6 @@ def window_error(normalized_rsme, labels_anomaly):
f1s = {}
for i in range(0,window_len):
window_arr = []
for j in range(0, len(normalized_rsme)):
if j < i:
s = np.sum(normalized_rsme[0:j+1])
......@@ -152,9 +158,9 @@ def window_error(normalized_rsme, labels_anomaly):
s = np.sum(normalized_rsme[j-i : j])
window_arr.append(s)
f1_max, preds, th_opt = get_th_opt(window_arr, labels_anomaly, True)
f1s[f1_max] = preds
print("F1MAX", max(f1s.items(), key=operator.itemgetter(0))[1])
return max(f1s.items(), key=operator.itemgetter(0))[1]
......@@ -193,7 +199,7 @@ def anomaly_houses_4():
print("------------NORMAL-MAE---------------------------------------")
for i in range(0,38):
df_house_anomaly = pd.read_csv(data_dir_path + '/normal_f_csv/house_{}_row.csv'.format(i), header=None)
df_house_anomaly = pd.read_csv(DATA_DIR_PATH + '/normal_f_csv/house_{}_row.csv'.format(i), header=None)
scaled_house_anomaly = scaler.transform(df_house_anomaly)
out_anomaly = ae.predict_mine(scaled_house_anomaly)
mae_anomaly = mean_absolute_error(scaled_house_anomaly, out_anomaly, multioutput='raw_values')
......@@ -203,7 +209,7 @@ def anomaly_houses_4():
print("-------------NORMAL-RSME--------------------------------------")
for i in range(0,38):
df_house_anomaly = pd.read_csv(data_dir_path + '/normal_f_csv/house_{}_row.csv'.format(i), header=None)
df_house_anomaly = pd.read_csv(DATA_DIR_PATH + '/normal_f_csv/house_{}_row.csv'.format(i), header=None)
scaled_house_anomaly = scaler.transform(df_house_anomaly)
out_anomaly = ae.predict_mine(scaled_house_anomaly)
rsme_anomaly = np.sqrt(mean_squared_error(scaled_house_anomaly, out_anomaly, multioutput='raw_values'))
......@@ -214,7 +220,7 @@ def anomaly_houses_4():
mae_values = []
for i in range(0,38):
df_house_anomaly = pd.read_csv(data_dir_path + '/anomaly_4_f_csv/house_{}_anomaly.csv'.format(i), header=None)
df_house_anomaly = pd.read_csv(DATA_DIR_PATH + '/anomaly_4_f_csv/house_{}_anomaly.csv'.format(i), header=None)
scaled_house_anomaly = scaler.transform(df_house_anomaly)
out_anomaly = ae.predict_mine(scaled_house_anomaly)
mae_anomaly = mean_absolute_error(scaled_house_anomaly, out_anomaly, multioutput='raw_values')
......@@ -229,7 +235,7 @@ def anomaly_houses_4():
rsme_values = []
for i in range(0,38):
df_house_anomaly = pd.read_csv(data_dir_path + '/anomaly_4_f_csv/house_{}_anomaly.csv'.format(i), header=None)
df_house_anomaly = pd.read_csv(DATA_DIR_PATH + '/anomaly_4_f_csv/house_{}_anomaly.csv'.format(i), header=None)
scaled_house_anomaly = scaler.transform(df_house_anomaly)
out_anomaly = ae.predict_mine(scaled_house_anomaly)
rsme_anomaly = np.sqrt(mean_squared_error(scaled_house_anomaly, out_anomaly, multioutput='raw_values'))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment