From ac801dc5acd819a845c88966c9dadd6da71f7c74 Mon Sep 17 00:00:00 2001 From: Lukas Prause Date: Thu, 20 Apr 2023 15:53:43 +0200 Subject: [PATCH] Changes for new modem. --- format_serial_txt_to_csv_EM9190.py | 52 ++++----- plot_single_transmission_EM9190.py | 177 +++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 33 deletions(-) create mode 100755 plot_single_transmission_EM9190.py diff --git a/format_serial_txt_to_csv_EM9190.py b/format_serial_txt_to_csv_EM9190.py index d0d9457..6df92b4 100755 --- a/format_serial_txt_to_csv_EM9190.py +++ b/format_serial_txt_to_csv_EM9190.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import csv import datetime import re from argparse import ArgumentParser @@ -7,7 +6,6 @@ import pandas as pd KEY_VALUE_REGEX = r"(.+):(.+)" - if __name__ == "__main__": parser = ArgumentParser() @@ -18,54 +16,42 @@ if __name__ == "__main__": content = file.read() file.close() - header = ["time"] - csv_lines = list() + serial_df = None p = re.compile(KEY_VALUE_REGEX) for part in content.split(";;;"): if part == "": break part = part.replace("\t", "\n").strip() - - csv_line = list() time = None + line_dict = dict(time=None) for line in part.split("\n"): if not line.startswith("!") or line == "" or line == "\n": - if time is None: + if line_dict["time"] is None: time = line - csv_line.append(time) + line_dict["time"] = [time] m = p.match(line) if m: key = m.group(1).strip().replace(" ", "_") - value = m.group(2).replace("MHz", "").strip() + value = m.group(2).replace("MHz", "").replace("---", "").strip() - if key not in header: - header.append(key) + line_dict[key] = [value] - csv_line.append(value) - if len(csv_line) > 1: - #print(csv_line) - csv_lines.append(csv_line) + if len(line_dict) > 1: + #print("line:") + #print(line_dict) + #print("serial_df:") + #print(serial_df) + if serial_df is None: + serial_df = pd.DataFrame.from_dict(line_dict, orient="columns",) + else: + serial_df = pd.concat([serial_df, pd.DataFrame.from_dict(line_dict, orient="columns")]) - - - outputfile = open(args.file.replace("txt", "csv"), "w") - writer = csv.writer(outputfile, delimiter=",", lineterminator="\n", escapechar='\\') - writer.writerow(header) - #print(all_csv_lines) - for l in csv_lines: - #print(l) - writer.writerow(l) - - outputfile.close() - - outputfile = open(args.file.replace("txt", "csv"), "r") - serial_df = pd.read_csv(outputfile) + serial_df = serial_df.copy() serial_df["datetime"] = pd.to_datetime( - serial_df["time"].apply(lambda x: datetime.datetime.fromtimestamp(x)) + serial_df["time"].apply(lambda x: datetime.datetime.fromtimestamp(int(x))) ) serial_df.to_csv(args.file.replace("txt", "csv")) - outputfile.close() - - + #serial_df = serial_df.filter(["datetime", "LTE_bw", "LTE_SCC2_bw", "LTE_SCC3_bw", "LTE_SCC4_bw", "SCC1_NR5G_bw", "NR5G_dl_bw", "NR5G_ul_bw", "LTE_SCC1_bw", "NR5G_bw"]) + #print(serial_df.to_string()) \ No newline at end of file diff --git a/plot_single_transmission_EM9190.py b/plot_single_transmission_EM9190.py new file mode 100755 index 0000000..0df44a0 --- /dev/null +++ b/plot_single_transmission_EM9190.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +import math +import multiprocessing +import os +from argparse import ArgumentParser + +import matplotlib +import pandas as pd +import matplotlib.pyplot as plt + +# Using seaborn's style +#plt.style.use('seaborn') + +tex_fonts = { + "pgf.texsystem": "lualatex", + # "legend.fontsize": "x-large", + # "figure.figsize": (15, 5), + "axes.labelsize": 15, # "small", + # "axes.titlesize": "x-large", + "xtick.labelsize": 15, # "small", + "ytick.labelsize": 15, # "small", + "legend.fontsize": 15, + "axes.formatter.use_mathtext": True, + "mathtext.fontset": "dejavusans", +} + +#plt.rcParams.update(tex_fonts) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("-s", "--serial_file", required=True, help="Serial csv file.") + parser.add_argument("-p", "--pcap_csv_folder", required=True, help="PCAP csv folder.") + parser.add_argument("--save", required=True, help="Location to save pdf file.") + parser.add_argument( + "-i", + "--interval", + default=10, + type=int, + help="Time interval for rolling window.", + ) + + args = parser.parse_args() + + pcap_csv_list = list() + for filename in os.listdir(args.pcap_csv_folder): + if filename.endswith(".csv") and "tcp" in filename: + pcap_csv_list.append(filename) + + counter = 1 + if len(pcap_csv_list) == 0: + print("No CSV files found.") + + pcap_csv_list.sort(key=lambda x: int(x.split("_")[-1].replace(".csv", ""))) + + for csv in pcap_csv_list: + + print("\rProcessing {} out of {} CSVs.\t({}%)\t".format(counter, len(pcap_csv_list), math.floor(counter/len(pcap_csv_list)))) + + #try: + transmission_df = pd.read_csv( + "{}{}".format(args.pcap_csv_folder, csv), + dtype=dict(is_retranmission=bool, is_dup_ack=bool), + ) + + transmission_df["datetime"] = pd.to_datetime(transmission_df["datetime"]) - pd.Timedelta(hours=1) + transmission_df = transmission_df.set_index("datetime") + transmission_df.index = pd.to_datetime(transmission_df.index) + transmission_df = transmission_df.sort_index() + + # srtt to [s] + transmission_df["srtt"] = transmission_df["srtt"].apply(lambda x: x / 10**6) + + # key for columns and level for index + transmission_df["goodput"] = transmission_df["payload_size"].groupby(pd.Grouper(level="datetime", freq="{}s".format(args.interval))).transform("sum") + transmission_df["goodput"] = transmission_df["goodput"].apply( + lambda x: ((x * 8) / args.interval) / 10**6 + ) + + transmission_df["goodput_rolling"] = transmission_df["payload_size"].rolling("{}s".format(args.interval)).sum() + transmission_df["goodput_rolling"] = transmission_df["goodput_rolling"].apply( + lambda x: ((x * 8) / args.interval) / 10 ** 6 + ) + + # set meta values and remove all not needed columns + cc_algo = transmission_df["congestion_control"].iloc[0] + cc_algo = cc_algo.upper() + transmission_direction = transmission_df["direction"].iloc[0] + + #transmission_df = transmission_df.filter(["goodput", "datetime", "ack_rtt", "goodput_rolling", "snd_cwnd"]) + + # read serial csv + serial_df = pd.read_csv(args.serial_file) + serial_df["datetime"] = pd.to_datetime(serial_df["datetime"]) - pd.Timedelta(hours=1) + serial_df = serial_df.set_index("datetime") + serial_df.index = pd.to_datetime(serial_df.index) + serial_df.sort_index() + + transmission_df = pd.merge_asof( + transmission_df, + serial_df, + tolerance=pd.Timedelta("1s"), + right_index=True, + left_index=True, + ) + + # sum bandwidth + # columns: LTE_bw, LTE_SCC2_bw, LTE_SCC3_bw, LTE_SCC4_bw, SCC1_NR5G_bw, NR5G_dl_bw, NR5G_ul_bw, LTE_SCC1_bw, NR5G_bw + transmission_df["bw_sum"] = transmission_df["LTE_bw"] + transmission_df["LTE_SCC2_bw"] \ + + transmission_df["LTE_SCC3_bw"] + transmission_df["LTE_SCC4_bw"] \ + + transmission_df["SCC1_NR5G_bw"] + transmission_df["NR5G_dl_bw"] \ + + transmission_df["LTE_SCC1_bw"] + + transmission_df.index = transmission_df["arrival_time"] + + # transmission timeline + scaley = 1.5 + scalex = 1.0 + plt.title("{} with {}".format(transmission_direction, cc_algo)) + fig, ax = plt.subplots(2, 1, figsize=[6.4 * scaley, 4.8 * scalex]) + fig.subplots_adjust(right=0.75) + fig.suptitle("{} with {}".format(transmission_direction, cc_algo)) + ax0 = ax[0] + ax1 = ax0.twinx() + ax2 = ax0.twinx() + #ax2.spines.right.set_position(("axes", 1.22)) + + ax00 = ax[1] + ax01 = ax00.twinx() + ax02 = ax00.twinx() + + # Plot vertical lines + lte_handovers = transmission_df["lte_pcid"].diff().dropna() + for index, value in lte_handovers.items(): + if value > 0: + ax00.axvline(index, ymin=0, ymax=1, color="skyblue", label="4G Handover") + + nr_handovers = transmission_df["nr_pcid"].diff().dropna() + for index, value in nr_handovers.items(): + if value > 0: + ax00.axvline(index, ymin=0, ymax=1, color="greenyellow", label="5G Handover") + + ax0.plot(transmission_df["snd_cwnd"].dropna(), color="lime", linestyle="dashed", label="cwnd") + ax1.plot(transmission_df["srtt"].dropna(), color="red", linestyle="dashdot", label="sRTT") + ax2.plot(transmission_df["goodput_rolling"], color="blue", linestyle="solid", label="goodput") + ax00.plot(transmission_df["NR5G_RSRQ_(dB)"].dropna(), color="magenta", linestyle="dotted", label="NR RSRQ") + ax01.plot(transmission_df["bw_sum"].dropna(), color="peru", linestyle="dotted", label="bandwidth") + ax02.plot(transmission_df["RSRQ_(dB)"].dropna(), color="magenta", linestyle="dotted", label="LTE RSRQ") + + ax2.spines.right.set_position(("axes", 1.1)) + + ax0.set_ylim(0, 5000) + ax1.set_ylim(0, 0.3) + ax2.set_ylim(0, 500) + ax00.set_ylim(-25, -5) + ax01.set_ylim(0, 200) + ax02.set_ylim(-25, -5) + + ax00.set_xlabel("arrival time [s]") + + ax2.set_ylabel("Goodput [mbps]") + ax00.set_ylabel("NR RSRQ [dB]") + ax02.set_ylabel("LTE RSRQ [dB]") + ax1.set_ylabel("sRTT [s]") + ax0.set_ylabel("cwnd") + ax01.set_ylabel("Bandwidth [MHz]") + + fig.legend(loc="lower right") + + plt.savefig("{}{}_plot.pdf".format(args.save, csv.replace(".csv", ""))) + #except Exception as e: + # print("Error processing file: {}".format(csv)) + # print(str(e)) + counter += 1 + + plt.close(fig) + plt.clf()