Plotting data related to gps loaction.
This commit is contained in:
Lukas Prause
208
plot_gps.py
Executable file
208
plot_gps.py
Executable file
@@ -0,0 +1,208 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import multiprocessing
|
||||||
|
import os
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from math import ceil
|
||||||
|
from time import sleep
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import geopandas as gpd
|
||||||
|
import contextily as cx
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
from util import chunk_list
|
||||||
|
|
||||||
|
|
||||||
|
def csv_to_dataframe(csv_list, dummy):
|
||||||
|
|
||||||
|
global n
|
||||||
|
global frame_list
|
||||||
|
|
||||||
|
transmission_df = None
|
||||||
|
|
||||||
|
for csv in csv_list:
|
||||||
|
tmp_df = pd.read_csv(
|
||||||
|
"{}{}".format(args.pcap_csv_folder, csv),
|
||||||
|
dtype=dict(is_retranmission=bool, is_dup_ack=bool),
|
||||||
|
)
|
||||||
|
tmp_df["datetime"] = pd.to_datetime(tmp_df["datetime"]) - pd.Timedelta(hours=1)
|
||||||
|
tmp_df = tmp_df.set_index("datetime")
|
||||||
|
tmp_df.index = pd.to_datetime(tmp_df.index)
|
||||||
|
if transmission_df is None:
|
||||||
|
transmission_df = tmp_df
|
||||||
|
else:
|
||||||
|
transmission_df = pd.concat([transmission_df, tmp_df])
|
||||||
|
|
||||||
|
n.value += 1
|
||||||
|
|
||||||
|
frame_list.append(transmission_df)
|
||||||
|
|
||||||
|
|
||||||
|
from itertools import islice
|
||||||
|
|
||||||
|
|
||||||
|
def chunk(it, size):
|
||||||
|
it = iter(it)
|
||||||
|
return iter(lambda: tuple(islice(it, size)), ())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument("-f", "--gps_file", required=True, help="GPS csv file.")
|
||||||
|
parser.add_argument("-s", "--serial_file", required=True, help="Serial csv file.")
|
||||||
|
parser.add_argument("-p", "--pcap_csv_folder", required=True, help="PCAP csv folder.")
|
||||||
|
parser.add_argument("-a", "--column", required=True, help="Column to plot")
|
||||||
|
parser.add_argument("-l", "--label", help="Label above the plot.")
|
||||||
|
parser.add_argument("--no_legend", action="store_false", default=True, help="Do not show legend.")
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--show_providerinfo",
|
||||||
|
default=False,
|
||||||
|
help="Show providerinfo for map tiles an zoom levels.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-c",
|
||||||
|
"--cores",
|
||||||
|
default=1,
|
||||||
|
type=int,
|
||||||
|
help="Number of cores for multiprocessing.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-i",
|
||||||
|
"--interval",
|
||||||
|
default=10,
|
||||||
|
type=int,
|
||||||
|
help="Time interval for rolling window.",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
manager = multiprocessing.Manager()
|
||||||
|
n = manager.Value("i", 0)
|
||||||
|
frame_list = manager.list()
|
||||||
|
jobs = []
|
||||||
|
|
||||||
|
# load all pcap csv into one dataframe
|
||||||
|
pcap_csv_list = list()
|
||||||
|
for filename in os.listdir(args.pcap_csv_folder):
|
||||||
|
if filename.endswith(".csv") and "tcp" in filename:
|
||||||
|
pcap_csv_list.append(filename)
|
||||||
|
|
||||||
|
parts = chunk(pcap_csv_list, ceil(len(pcap_csv_list) / args.cores))
|
||||||
|
print("Start processing with {} jobs.".format(args.cores))
|
||||||
|
for p in parts:
|
||||||
|
process = multiprocessing.Process(target=csv_to_dataframe, args=(p, "dummy"))
|
||||||
|
jobs.append(process)
|
||||||
|
|
||||||
|
for j in jobs:
|
||||||
|
j.start()
|
||||||
|
|
||||||
|
print("Started all jobs.")
|
||||||
|
# Ensure all of the processes have finished
|
||||||
|
finished_job_counter = 0
|
||||||
|
working = ["|", "/", "-", "\\", "|", "/", "-", "\\"]
|
||||||
|
w = 0
|
||||||
|
while len(jobs) != finished_job_counter:
|
||||||
|
sleep(1)
|
||||||
|
print(
|
||||||
|
"\r\t{}{}{}\t Running {} jobs ({} finished). Processed {} out of {} pcap csv files. ({}%) ".format(
|
||||||
|
working[w],
|
||||||
|
working[w],
|
||||||
|
working[w],
|
||||||
|
len(jobs),
|
||||||
|
finished_job_counter,
|
||||||
|
n.value,
|
||||||
|
len(pcap_csv_list),
|
||||||
|
round((n.value / len(pcap_csv_list)) * 100, 2),
|
||||||
|
),
|
||||||
|
end="",
|
||||||
|
)
|
||||||
|
finished_job_counter = 0
|
||||||
|
for j in jobs:
|
||||||
|
if not j.is_alive():
|
||||||
|
finished_job_counter += 1
|
||||||
|
if (w + 1) % len(working) == 0:
|
||||||
|
w = 0
|
||||||
|
else:
|
||||||
|
w += 1
|
||||||
|
print("\r\nSorting table...")
|
||||||
|
|
||||||
|
transmission_df = pd.concat(frame_list)
|
||||||
|
frame_list = None
|
||||||
|
transmission_df = transmission_df.sort_index()
|
||||||
|
|
||||||
|
print("Calculate goodput...")
|
||||||
|
transmission_df["goodput"] = transmission_df["payload_size"].rolling("{}s".format(args.interval)).sum()
|
||||||
|
transmission_df["goodput"] = transmission_df["goodput"].apply(
|
||||||
|
lambda x: ((x * 8) / args.interval) / 10**6
|
||||||
|
)
|
||||||
|
|
||||||
|
# load dataframe an put it into geopandas
|
||||||
|
df = pd.read_csv(args.gps_file)
|
||||||
|
df["kmh"] = df["speed (knots)"].apply(lambda x: x * 1.852)
|
||||||
|
df = df.set_index("datetime")
|
||||||
|
df.index = pd.to_datetime(df.index)
|
||||||
|
|
||||||
|
gdf = gpd.GeoDataFrame(
|
||||||
|
df,
|
||||||
|
geometry=gpd.points_from_xy(df["longitude"], df["latitude"]),
|
||||||
|
crs="EPSG:4326",
|
||||||
|
)
|
||||||
|
gdf = pd.merge_asof(
|
||||||
|
gdf,
|
||||||
|
transmission_df,
|
||||||
|
tolerance=pd.Timedelta("10s"),
|
||||||
|
right_index=True,
|
||||||
|
left_index=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# read serial csv
|
||||||
|
serial_df = pd.read_csv(args.serial_file)
|
||||||
|
serial_df["datetime"] = pd.to_datetime(serial_df["datetime"]) - pd.Timedelta(hours=1)
|
||||||
|
serial_df = serial_df.set_index("datetime")
|
||||||
|
serial_df.index = pd.to_datetime(serial_df.index)
|
||||||
|
|
||||||
|
gdf = pd.merge_asof(
|
||||||
|
gdf,
|
||||||
|
serial_df,
|
||||||
|
tolerance=pd.Timedelta("1s"),
|
||||||
|
right_index=True,
|
||||||
|
left_index=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(gdf)
|
||||||
|
|
||||||
|
print("Start plotting...")
|
||||||
|
# format to needed format and add basemap as background
|
||||||
|
df_wm = gdf.to_crs(epsg=3857)
|
||||||
|
#df_wm.to_csv("debug-data.csv")
|
||||||
|
# ax2 = df_wm.plot(figsize=(10, 10), alpha=0.5, edgecolor='k')
|
||||||
|
print(df_wm)
|
||||||
|
ax2 = df_wm.plot()
|
||||||
|
ax2 = df_wm.plot(args.column, cmap="hot", legend=args.no_legend, ax=ax2)
|
||||||
|
# ax2 = df_wm.plot.scatter(x="longitude", y="latitude", c="kmh", cmap="hot")
|
||||||
|
# zoom 17 is pretty
|
||||||
|
cx.add_basemap(ax2, source=cx.providers.OpenStreetMap.Mapnik, zoom=15)
|
||||||
|
|
||||||
|
# gdf.plot()
|
||||||
|
ax2.set_axis_off()
|
||||||
|
ax2.set_title(args.label if args.label else args.column)
|
||||||
|
|
||||||
|
if args.show_providerinfo:
|
||||||
|
#####################################
|
||||||
|
# Identifying how many tiles
|
||||||
|
latlon_outline = gdf.to_crs("epsg:4326").total_bounds
|
||||||
|
def_zoom = cx.tile._calculate_zoom(*latlon_outline)
|
||||||
|
print(f"Default Zoom level {def_zoom}")
|
||||||
|
|
||||||
|
cx.howmany(*latlon_outline, def_zoom, ll=True)
|
||||||
|
cx.howmany(*latlon_outline, def_zoom + 1, ll=True)
|
||||||
|
cx.howmany(*latlon_outline, def_zoom + 2, ll=True)
|
||||||
|
|
||||||
|
# Checking out some of the other providers and tiles
|
||||||
|
print(cx.providers.CartoDB.Voyager)
|
||||||
|
print(cx.providers.Stamen.TonerLite)
|
||||||
|
print(cx.providers.Stamen.keys())
|
||||||
|
#####################################
|
||||||
|
|
||||||
|
# df.plot(x="longitude", y="latitude", kind="scatter", colormap="YlOrRd")
|
||||||
|
plt.show()
|
||||||
Reference in New Issue
Block a user