Changes for new modem.

format_serial_txt_to_csv_EM9190.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import csv
 import datetime
 import re
 from argparse import ArgumentParser
@@ -7,7 +6,6 @@ import pandas as pd
 
 KEY_VALUE_REGEX = r"(.+):(.+)"
 
-
 if __name__ == "__main__":
 
     parser = ArgumentParser()
@@ -18,54 +16,42 @@ if __name__ == "__main__":
     content = file.read()
     file.close()
 
-    header = ["time"]
+    serial_df = None
-    csv_lines = list()
     p = re.compile(KEY_VALUE_REGEX)
 
     for part in content.split(";;;"):
         if part == "":
             break
         part = part.replace("\t", "\n").strip()
-
-        csv_line = list()
         time = None
+        line_dict = dict(time=None)
         for line in part.split("\n"):
             if not line.startswith("!") or line == "" or line == "\n":
-                if time is None:
+                if line_dict["time"] is None:
                     time = line
-                    csv_line.append(time)
+                    line_dict["time"] = [time]
                 m = p.match(line)
                 if m:
                     key = m.group(1).strip().replace(" ", "_")
-                    value = m.group(2).replace("MHz", "").strip()
+                    value = m.group(2).replace("MHz", "").replace("---", "").strip()
 
-                    if key not in header:
+                    line_dict[key] = [value]
-                        header.append(key)
 
-                    csv_line.append(value)
+        if len(line_dict) > 1:
-        if len(csv_line) > 1:
+            #print("line:")
-            #print(csv_line)
+            #print(line_dict)
-            csv_lines.append(csv_line)
+            #print("serial_df:")
+            #print(serial_df)
+            if serial_df is None:
+                serial_df = pd.DataFrame.from_dict(line_dict, orient="columns",)
+            else:
+                serial_df = pd.concat([serial_df, pd.DataFrame.from_dict(line_dict, orient="columns")])
 
-
+    serial_df = serial_df.copy()
-
-    outputfile = open(args.file.replace("txt", "csv"), "w")
-    writer = csv.writer(outputfile, delimiter=",", lineterminator="\n", escapechar='\\')
-    writer.writerow(header)
-    #print(all_csv_lines)
-    for l in csv_lines:
-        #print(l)
-        writer.writerow(l)
-
-    outputfile.close()
-
-    outputfile = open(args.file.replace("txt", "csv"), "r")
-    serial_df = pd.read_csv(outputfile)
     serial_df["datetime"] = pd.to_datetime(
-        serial_df["time"].apply(lambda x: datetime.datetime.fromtimestamp(x))
+        serial_df["time"].apply(lambda x: datetime.datetime.fromtimestamp(int(x)))
     )
     serial_df.to_csv(args.file.replace("txt", "csv"))
-    outputfile.close()
-
-
 
+    #serial_df = serial_df.filter(["datetime", "LTE_bw", "LTE_SCC2_bw", "LTE_SCC3_bw", "LTE_SCC4_bw", "SCC1_NR5G_bw", "NR5G_dl_bw", "NR5G_ul_bw", "LTE_SCC1_bw", "NR5G_bw"])
+    #print(serial_df.to_string())

plot_single_transmission_EM9190.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+import math
+import multiprocessing
+import os
+from argparse import ArgumentParser
+
+import matplotlib
+import pandas as pd
+import matplotlib.pyplot as plt
+
+# Using seaborn's style
+#plt.style.use('seaborn')
+
+tex_fonts = {
+    "pgf.texsystem": "lualatex",
+    # "legend.fontsize": "x-large",
+    # "figure.figsize": (15, 5),
+    "axes.labelsize": 15,  # "small",
+    # "axes.titlesize": "x-large",
+    "xtick.labelsize": 15,  # "small",
+    "ytick.labelsize": 15,  # "small",
+    "legend.fontsize": 15,
+    "axes.formatter.use_mathtext": True,
+    "mathtext.fontset": "dejavusans",
+}
+
+#plt.rcParams.update(tex_fonts)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("-s", "--serial_file", required=True, help="Serial csv file.")
+    parser.add_argument("-p", "--pcap_csv_folder", required=True, help="PCAP csv folder.")
+    parser.add_argument("--save", required=True, help="Location to save pdf file.")
+    parser.add_argument(
+        "-i",
+        "--interval",
+        default=10,
+        type=int,
+        help="Time interval for rolling window.",
+    )
+
+    args = parser.parse_args()
+
+    pcap_csv_list = list()
+    for filename in os.listdir(args.pcap_csv_folder):
+        if filename.endswith(".csv") and "tcp" in filename:
+            pcap_csv_list.append(filename)
+
+    counter = 1
+    if len(pcap_csv_list) == 0:
+        print("No CSV files found.")
+
+    pcap_csv_list.sort(key=lambda x: int(x.split("_")[-1].replace(".csv", "")))
+
+    for csv in pcap_csv_list:
+
+        print("\rProcessing {} out of {} CSVs.\t({}%)\t".format(counter, len(pcap_csv_list), math.floor(counter/len(pcap_csv_list))))
+
+        #try:
+        transmission_df = pd.read_csv(
+            "{}{}".format(args.pcap_csv_folder, csv),
+            dtype=dict(is_retranmission=bool, is_dup_ack=bool),
+        )
+
+        transmission_df["datetime"] = pd.to_datetime(transmission_df["datetime"]) - pd.Timedelta(hours=1)
+        transmission_df = transmission_df.set_index("datetime")
+        transmission_df.index = pd.to_datetime(transmission_df.index)
+        transmission_df = transmission_df.sort_index()
+
+        # srtt to [s]
+        transmission_df["srtt"] = transmission_df["srtt"].apply(lambda x: x / 10**6)
+
+        # key for columns and level for index
+        transmission_df["goodput"] = transmission_df["payload_size"].groupby(pd.Grouper(level="datetime", freq="{}s".format(args.interval))).transform("sum")
+        transmission_df["goodput"] = transmission_df["goodput"].apply(
+            lambda x: ((x * 8) / args.interval) / 10**6
+        )
+
+        transmission_df["goodput_rolling"] = transmission_df["payload_size"].rolling("{}s".format(args.interval)).sum()
+        transmission_df["goodput_rolling"] = transmission_df["goodput_rolling"].apply(
+            lambda x: ((x * 8) / args.interval) / 10 ** 6
+        )
+
+        # set meta values and remove all not needed columns
+        cc_algo = transmission_df["congestion_control"].iloc[0]
+        cc_algo = cc_algo.upper()
+        transmission_direction = transmission_df["direction"].iloc[0]
+
+        #transmission_df = transmission_df.filter(["goodput", "datetime", "ack_rtt", "goodput_rolling", "snd_cwnd"])
+
+        # read serial csv
+        serial_df = pd.read_csv(args.serial_file)
+        serial_df["datetime"] = pd.to_datetime(serial_df["datetime"]) - pd.Timedelta(hours=1)
+        serial_df = serial_df.set_index("datetime")
+        serial_df.index = pd.to_datetime(serial_df.index)
+        serial_df.sort_index()
+
+        transmission_df = pd.merge_asof(
+            transmission_df,
+            serial_df,
+            tolerance=pd.Timedelta("1s"),
+            right_index=True,
+            left_index=True,
+        )
+
+        # sum bandwidth
+        # columns: LTE_bw, LTE_SCC2_bw, LTE_SCC3_bw, LTE_SCC4_bw, SCC1_NR5G_bw, NR5G_dl_bw, NR5G_ul_bw, LTE_SCC1_bw, NR5G_bw
+        transmission_df["bw_sum"] = transmission_df["LTE_bw"] + transmission_df["LTE_SCC2_bw"] \
+                                    + transmission_df["LTE_SCC3_bw"] + transmission_df["LTE_SCC4_bw"] \
+                                    + transmission_df["SCC1_NR5G_bw"] + transmission_df["NR5G_dl_bw"] \
+                                    + transmission_df["LTE_SCC1_bw"]
+
+        transmission_df.index = transmission_df["arrival_time"]
+
+        # transmission timeline
+        scaley = 1.5
+        scalex = 1.0
+        plt.title("{} with {}".format(transmission_direction, cc_algo))
+        fig, ax = plt.subplots(2, 1, figsize=[6.4 * scaley, 4.8 * scalex])
+        fig.subplots_adjust(right=0.75)
+        fig.suptitle("{} with {}".format(transmission_direction, cc_algo))
+        ax0 = ax[0]
+        ax1 = ax0.twinx()
+        ax2 = ax0.twinx()
+        #ax2.spines.right.set_position(("axes", 1.22))
+
+        ax00 = ax[1]
+        ax01 = ax00.twinx()
+        ax02 = ax00.twinx()
+
+        # Plot vertical lines
+        lte_handovers = transmission_df["lte_pcid"].diff().dropna()
+        for index, value in lte_handovers.items():
+            if value > 0:
+                ax00.axvline(index, ymin=0, ymax=1, color="skyblue", label="4G Handover")
+
+        nr_handovers = transmission_df["nr_pcid"].diff().dropna()
+        for index, value in nr_handovers.items():
+            if value > 0:
+                ax00.axvline(index, ymin=0, ymax=1, color="greenyellow", label="5G Handover")
+
+        ax0.plot(transmission_df["snd_cwnd"].dropna(), color="lime", linestyle="dashed", label="cwnd")
+        ax1.plot(transmission_df["srtt"].dropna(), color="red", linestyle="dashdot", label="sRTT")
+        ax2.plot(transmission_df["goodput_rolling"], color="blue", linestyle="solid", label="goodput")
+        ax00.plot(transmission_df["NR5G_RSRQ_(dB)"].dropna(), color="magenta", linestyle="dotted", label="NR RSRQ")
+        ax01.plot(transmission_df["bw_sum"].dropna(), color="peru", linestyle="dotted", label="bandwidth")
+        ax02.plot(transmission_df["RSRQ_(dB)"].dropna(), color="magenta", linestyle="dotted", label="LTE RSRQ")
+
+        ax2.spines.right.set_position(("axes", 1.1))
+
+        ax0.set_ylim(0, 5000)
+        ax1.set_ylim(0, 0.3)
+        ax2.set_ylim(0, 500)
+        ax00.set_ylim(-25, -5)
+        ax01.set_ylim(0, 200)
+        ax02.set_ylim(-25, -5)
+
+        ax00.set_xlabel("arrival time [s]")
+        
+        ax2.set_ylabel("Goodput [mbps]")
+        ax00.set_ylabel("NR RSRQ [dB]")
+        ax02.set_ylabel("LTE RSRQ [dB]")
+        ax1.set_ylabel("sRTT [s]")
+        ax0.set_ylabel("cwnd")
+        ax01.set_ylabel("Bandwidth [MHz]")
+
+        fig.legend(loc="lower right")
+
+        plt.savefig("{}{}_plot.pdf".format(args.save, csv.replace(".csv", "")))
+        #except Exception as e:
+        #    print("Error processing file: {}".format(csv))
+        #    print(str(e))
+        counter += 1
+
+        plt.close(fig)
+        plt.clf()