diff --git a/src/tracely/clean_trace.py b/src/tracely/clean_trace.py index b83b5c3..8edec99 100644 --- a/src/tracely/clean_trace.py +++ b/src/tracely/clean_trace.py @@ -9,6 +9,7 @@ from .create_trace_data import CreateTraceData from .utils.data_validation_utils import DataValidationUtils from .utils.utils import get_haversine_distance, \ + vectorized_haversine, \ calculate_change_in_direction, \ convert_unix_timestamp_to_human_readable, \ convert_time_interval_to_human_readable, \ @@ -71,7 +72,7 @@ def __init__(self, self.trace_data = CreateTraceData(input_trace_payload).create_trace_data() self.vehicle_type = self.trace_data["vehicle_type"] self.vehicle_speed = self.trace_data["vehicle_speed"] # km/hr - self.trace_df = copy.deepcopy(self.trace_data["trace_df"]) + self.trace_df = self.trace_data["trace_df"].copy() self.raw_trace = self.trace_df.to_dict(orient="records") self.input_pings_count = len(self.trace_df) self.elapsed_time_record = [] @@ -213,49 +214,48 @@ def remove_nearby_pings(self, # Validate parameter DataValidationUtils.validate_remove_nearby_pings_parameters(min_dist_bw_consecutive_pings=min_dist_bw_consecutive_pings) + # Extract columns as numpy arrays for fast access + latitudes = self.trace_df["cleaned_latitude"].values + longitudes = self.trace_df["cleaned_longitude"].values + update_statuses = self.trace_df["update_status"].values + force_retains = self.trace_df["force_retain"].values + # Get the first ping as the starting reference - prev_ping = (self.trace_df.iloc[0]["cleaned_latitude"],self.trace_df.iloc[0]["cleaned_longitude"]) + prev_lat, prev_lng = latitudes[0], longitudes[0] # Create a mask to store which rows need to be updated mask = np.zeros(len(self.trace_df), dtype=bool) - for i, row in enumerate(self.trace_df.to_dict("records")[1:], 1): - - current_ping = (row["cleaned_latitude"], row["cleaned_longitude"]) + for i in range(1, len(self.trace_df)): - if ((prev_ping[0] is None) or - (prev_ping[1] is None) or - (pd.isna(prev_ping[0])) or - (pd.isna(prev_ping[1]))): + curr_lat, curr_lng = latitudes[i], longitudes[i] - prev_ping = current_ping + if (prev_lat is None or prev_lng is None or + pd.isna(prev_lat) or pd.isna(prev_lng)): + prev_lat, prev_lng = curr_lat, curr_lng continue - if ((current_ping[0] is None) or - (current_ping[1] is None) or - (pd.isna(current_ping[0])) or - (pd.isna(current_ping[1]))): + if (curr_lat is None or curr_lng is None or + pd.isna(curr_lat) or pd.isna(curr_lng)): continue # Do not update current ping if it is an interpolated ping - if row["update_status"] == "interpolated": + if update_statuses[i] == "interpolated": continue # Calculate the distance between the current ping and the previous ping - distance = get_haversine_distance(prev_ping[0], - prev_ping[1], - current_ping[0], - current_ping[1]) + distance = get_haversine_distance(prev_lat, prev_lng, + curr_lat, curr_lng) if distance is None: continue - if (distance < min_dist_bw_consecutive_pings) and not (row["force_retain"]): + if (distance < min_dist_bw_consecutive_pings) and not force_retains[i]: mask[i] = True continue # Update the previous ping if the current one is kept - prev_ping = current_ping + prev_lat, prev_lng = curr_lat, curr_lng self.trace_df.loc[mask, ["cleaned_latitude", "cleaned_longitude", "update_status", "last_updated_by"]] = [None, None, "dropped", "remove_nearby_pings"] @@ -281,9 +281,9 @@ def _impute_distorted_pings_with_distance(self, Defaults to 1. """ - trace_df = copy.deepcopy(self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & - (~self.trace_df["cleaned_longitude"].isnull())]) - + trace_df = self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & + (~self.trace_df["cleaned_longitude"].isnull())].copy() + trace_df = trace_df.reset_index(drop=True) # Extract latitude, longitude and update_status data as NumPy array for faster access @@ -389,8 +389,8 @@ def impute_distorted_pings_with_angle(self, # Validate parameter DataValidationUtils.validate_impute_distorted_pings_with_angle_parameters(max_delta_angle=max_delta_angle) - trace_df = copy.deepcopy(self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & - (~self.trace_df["cleaned_longitude"].isnull())]) + trace_df = self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & + (~self.trace_df["cleaned_longitude"].isnull())].copy() trace_df = trace_df.reset_index(drop=True) # Convert DataFrame to NumPy array for faster access @@ -472,7 +472,7 @@ def map_match_trace(self, avg_snap_distance=avg_snap_distance, max_matched_dist_to_raw_dist_ratio=max_matched_dist_to_raw_dist_ratio) - trace_df = copy.deepcopy(self.trace_df) + trace_df = self.trace_df.copy() trace_segments = create_segments(trace_df, ping_batch_size) matched_segments = process_trace_segments(segments=trace_segments, osrm_url=osrm_url, @@ -699,13 +699,13 @@ def _get_stop_info(self, cumulative_stop_event_time, stop_event_sequence_number columns in the order they are written. """ - trace_df_copy = copy.deepcopy(trace_df) + trace_df_copy = trace_df.copy() # Create label key trace_df_copy["label_key"] = labels # Adding status key on the basis of labels in trace_df_copy - trace_df_copy["stop_event_status"] = trace_df_copy["label_key"].apply(lambda x: True if x >= 0 else False) + trace_df_copy["stop_event_status"] = trace_df_copy["label_key"] >= 0 # Adding representative ping for each batch of stopping ping stopping_pings_df = trace_df_copy[trace_df_copy["label_key"] != -1] @@ -747,9 +747,10 @@ def _get_stop_info(self, common_keys=["ping_id","cumulative_stop_event_time","stop_event_sequence_number"]) trace_df_copy["cumulative_stop_event_time"].replace({np.nan: 0}, inplace=True) - trace_df_copy["cumulative_stop_event_time"] = trace_df_copy.apply(lambda row: convert_time_interval_to_human_readable(row["cumulative_stop_event_time"], - format="ms"), - axis=1) + trace_df_copy["cumulative_stop_event_time"] = [ + convert_time_interval_to_human_readable(t, format="ms") + for t in trace_df_copy["cumulative_stop_event_time"].values + ] return trace_df_copy[["ping_id", "stop_event_status", "representative_stop_event_latitude", "representative_stop_event_longitude", "cumulative_stop_event_time", "stop_event_sequence_number"]] @@ -789,9 +790,9 @@ def add_stop_events_info(self, min_size=min_size, min_staying_time=min_staying_time) - trace_df = copy.deepcopy(self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & - (~self.trace_df["cleaned_longitude"].isnull()) & - (~self.trace_df["timestamp"].isnull())]) + trace_df = self.trace_df[(~self.trace_df["cleaned_latitude"].isnull()) & + (~self.trace_df["cleaned_longitude"].isnull()) & + (~self.trace_df["timestamp"].isnull())].copy() trace_df["timestamp"] = trace_df["timestamp"] // 1000 trace_list = trace_df[["cleaned_latitude","cleaned_longitude", "timestamp"]].values.tolist() @@ -804,8 +805,8 @@ def add_stop_events_info(self, stop_info_df = self._get_stop_info(trace_df,labels) - stop_info_df = self._merge_dataframes(copy.deepcopy(self.trace_df), - stop_info_df, "ping_id", + stop_info_df = self._merge_dataframes(self.trace_df.copy(), + stop_info_df, "ping_id", common_keys=constants.STOP_INFO_KEYS) stop_info_df["representative_stop_event_latitude"].replace({np.nan: None}, inplace=True) @@ -826,8 +827,7 @@ def _add_time_interval_bw_pings(self, pandas.Series: Series of time difference between consecutive pings. """ - timestamp_series = copy.deepcopy(trace_df[["timestamp"]]) - return timestamp_series["timestamp"].diff() + return trace_df["timestamp"].diff() def _add_distance_bw_pings(self, input_trace_df: pd.DataFrame) -> pd.Series: @@ -841,25 +841,29 @@ def _add_distance_bw_pings(self, pandas.Series: Series of difference in distance between consecutive pings. """ - trace_df = copy.deepcopy(input_trace_df[["cleaned_latitude", "cleaned_longitude"]]) + cleaned_lat = input_trace_df["cleaned_latitude"].values.astype(np.float64) + cleaned_lng = input_trace_df["cleaned_longitude"].values.astype(np.float64) + + # Step 1: Shift to get previous coordinates + prev_lat = np.empty_like(cleaned_lat) + prev_lat[0] = np.nan + prev_lat[1:] = cleaned_lat[:-1] - # Step 1: Shift the "cleaned_latitude", "cleaned_longitude" column to create the initial "previous_latitude", "previous_longitude" column - trace_df["previous_latitude"] = trace_df["cleaned_latitude"].shift(1) - trace_df["previous_longitude"] = trace_df["cleaned_longitude"].shift(1) + prev_lng = np.empty_like(cleaned_lng) + prev_lng[0] = np.nan + prev_lng[1:] = cleaned_lng[:-1] - # Step 2: Forward fill "previous_latitude", "previous_longitude" only where it is not NaN - trace_df["previous_latitude"] = trace_df["previous_latitude"].ffill() - trace_df["previous_longitude"] = trace_df["previous_longitude"].ffill() + # Step 2: Forward fill previous coordinates + prev_lat = pd.Series(prev_lat).ffill().values + prev_lng = pd.Series(prev_lng).ffill().values - # Step 3: Ensure "previous_latitude", "previous_longitude" are NaN where "cleaned_latitude", "cleaned_longitude" is NaN - trace_df["previous_latitude"] = trace_df.apply(lambda row: row["previous_latitude"] if pd.notna(row["cleaned_latitude"]) else np.nan, axis=1) - trace_df["previous_longitude"] = trace_df.apply(lambda row: row["previous_longitude"] if pd.notna(row["cleaned_longitude"]) else np.nan, axis=1) + # Step 3: Set previous to NaN where current is NaN + current_null = np.isnan(cleaned_lat) | np.isnan(cleaned_lng) + prev_lat[current_null] = np.nan + prev_lng[current_null] = np.nan - return trace_df.apply(lambda row: get_haversine_distance(row["previous_latitude"], - row["previous_longitude"], - row["cleaned_latitude"], - row["cleaned_longitude"]), - axis=1) + # Step 4: Vectorized haversine distance + return pd.Series(vectorized_haversine(prev_lat, prev_lng, cleaned_lat, cleaned_lng)) def _match_metadata(self, ping_list: list) -> list: @@ -988,19 +992,19 @@ def _create_output_distance_summary(self) -> dict: """ # Create copy of dataframe (only input_latitude, input_longitude columns) with only not null pings. - trace_lat_lng_df = copy.deepcopy(self.trace_df[["input_latitude", "input_longitude"]]) - trace_lat_lng_df.dropna(inplace=True) + trace_lat_lng_df = self.trace_df[["input_latitude", "input_longitude"]].dropna().copy() - # Add previous ping for each ping - trace_lat_lng_df["previous_latitude"] = trace_lat_lng_df["input_latitude"].shift(1) - trace_lat_lng_df["previous_longitude"] = trace_lat_lng_df["input_longitude"].shift(1) + # Vectorized distance computation using shifted coordinates + lat = trace_lat_lng_df["input_latitude"].values.astype(np.float64) + lng = trace_lat_lng_df["input_longitude"].values.astype(np.float64) + prev_lat = np.empty_like(lat) + prev_lat[0] = np.nan + prev_lat[1:] = lat[:-1] + prev_lng = np.empty_like(lng) + prev_lng[0] = np.nan + prev_lng[1:] = lng[:-1] - # Add distance of each ping from its previous ping - trace_lat_lng_df["dist_from_prev_ping"] = trace_lat_lng_df.apply(lambda row: get_haversine_distance(row["previous_latitude"], - row["previous_longitude"], - row["input_latitude"], - row["input_longitude"]), - axis=1) + trace_lat_lng_df["dist_from_prev_ping"] = vectorized_haversine(prev_lat, prev_lng, lat, lng) # Calculate distance of raw and clean trace # Use float type conversion to avoid numpy.int64 or numpy.float64 data type @@ -1044,7 +1048,7 @@ def _create_output_stop_summary(self) -> dict: "stop_events_info": [], "global_stop_events_info": {}} - df = copy.deepcopy(self.trace_df) + df = self.trace_df # Variables to track global stopping metrics total_trace_time = df["timestamp"].max()//1000 - df["timestamp"].min()//1000 @@ -1293,4 +1297,4 @@ def plot_raw_vs_stop_comparison_map(self) -> fl.plugins.DualMap: # Create and return map plot trace_output = self.get_trace_cleaning_output() trace = trace_output["cleaned_trace"] - return plot_stop_comparison_map(copy.deepcopy(self.raw_trace), trace, map_object) + return plot_stop_comparison_map(list(self.raw_trace), trace, map_object) diff --git a/src/tracely/utils/plotting_utils.py b/src/tracely/utils/plotting_utils.py index cfbc1f7..7afc6b2 100644 --- a/src/tracely/utils/plotting_utils.py +++ b/src/tracely/utils/plotting_utils.py @@ -71,50 +71,45 @@ def plot_raw_trace_from_trace_output(trace: list, map_object.add_child(child=raw_trace_segment_layer) layers_dict[segment_id] = raw_trace_segment_layer - # Extract latitude and longitude from trace_df and store as tuples in raw_trace_geometry + # Extract columns as arrays for fast access raw_trace_geometry = list(zip(trace_df["input_latitude"], trace_df["input_longitude"])) - - for index, row in trace_df.iterrows(): - - segment_id = row["trace_segment"] - segment_layer = layers_dict[segment_id] - - if (index != (len(trace_df) - 1)): - start = raw_trace_geometry[index] - end = raw_trace_geometry[index + 1] - fl.PolyLine(locations=[start, end], - color="red", - weight=3, - opacity=1, - tags=["raw"], + segments = trace_df["trace_segment"].values + ping_ids = trace_df["ping_id"].values + input_lats = trace_df["input_latitude"].values + input_lngs = trace_df["input_longitude"].values + timestamps = trace_df["timestamp"].values + error_radii = trace_df["error_radius"].values + event_types = trace_df["event_type"].values + force_retains = trace_df["force_retain"].values + n_rows = len(trace_df) + + for i in range(n_rows): + segment_layer = layers_dict[segments[i]] + + if i != (n_rows - 1): + fl.PolyLine(locations=[raw_trace_geometry[i], raw_trace_geometry[i + 1]], + color="red", weight=3, opacity=1, tags=["raw"], ).add_to(segment_layer) - icon = plugins.BeautifyIcon(icon_shape="circle", - number=index + 1, - border_color="blue", - background_color="transparent", + icon = plugins.BeautifyIcon(icon_shape="circle", number=i + 1, + border_color="blue", background_color="transparent", inner_icon_style="font-size:12px;") - raw_coordinates = [row["input_latitude"], - row["input_longitude"]] - - fl.Marker(raw_coordinates, + fl.Marker([input_lats[i], input_lngs[i]], icon=icon, - tooltip="Index: " + str(index + 1) + - "_ping_id: " + row["ping_id"], + tooltip="Index: " + str(i + 1) + "_ping_id: " + str(ping_ids[i]), tags=[""], - popup=create_general_popup({"ping_id": row["ping_id"], - "input_latitude": None if pd.isna(row["input_latitude"]) else row["input_latitude"], - "input_longitude": None if pd.isna(row["input_longitude"]) else row["input_longitude"], - "timestamp": int(row["timestamp"]), - "time_string": convert_unix_timestamp_to_human_readable(int((row["timestamp"])//1000)), - "error_radius": None if pd.isna(row["error_radius"]) else row["error_radius"], - "event_type": None if pd.isna(row["event_type"]) else row["event_type"], - "force_retain": row["force_retain"]} - ) + popup=create_general_popup({"ping_id": ping_ids[i], + "input_latitude": None if pd.isna(input_lats[i]) else input_lats[i], + "input_longitude": None if pd.isna(input_lngs[i]) else input_lngs[i], + "timestamp": int(timestamps[i]), + "time_string": convert_unix_timestamp_to_human_readable(int(timestamps[i]//1000)), + "error_radius": None if pd.isna(error_radii[i]) else error_radii[i], + "event_type": None if pd.isna(event_types[i]) else event_types[i], + "force_retain": force_retains[i]}) ).add_to(segment_layer) - - fl.LayerControl().add_to(map_object) + + fl.LayerControl().add_to(map_object) return map_object @@ -156,68 +151,54 @@ def plot_clean_trace_from_trace_output(trace, map_object.add_child(child=clean_trace_segment_layer) layers_dict[segment_id] = clean_trace_segment_layer - for index, row in trace_df.iterrows(): - - if ((row["cleaned_latitude"] is None) or - (row["cleaned_longitude"] is None) or - (pd.isna(row["cleaned_latitude"])) or - (pd.isna(row["cleaned_longitude"]))): - continue - - clean_trace_geometry.append((row["cleaned_latitude"],row["cleaned_longitude"])) + # Build geometry from non-null pings using vectorized filter + non_null_mask = trace_df["cleaned_latitude"].notna() & trace_df["cleaned_longitude"].notna() + non_null_df = trace_df[non_null_mask].reset_index(drop=True) + clean_lats = non_null_df["cleaned_latitude"].values + clean_lngs = non_null_df["cleaned_longitude"].values + clean_trace_geometry = list(zip(clean_lats, clean_lngs)) total_non_null_pings = len(clean_trace_geometry) - counter = 0 - - for index, row in trace_df.iterrows(): - if ((row["cleaned_latitude"] is None) or - (row["cleaned_longitude"] is None) or - (pd.isna(row["cleaned_latitude"])) or - (pd.isna(row["cleaned_longitude"]))): - continue - - segment_id = row["trace_segment"] - segment_layer = layers_dict[segment_id] - - if (counter != (total_non_null_pings - 1)): - start = clean_trace_geometry[counter] - end = clean_trace_geometry[counter + 1] - fl.PolyLine(locations=[start, end], - color="red", - weight=3, - opacity=1, - tags=["clean"], + segments = non_null_df["trace_segment"].values + ping_ids = non_null_df["ping_id"].values + input_lats = non_null_df["input_latitude"].values + input_lngs = non_null_df["input_longitude"].values + timestamps = non_null_df["timestamp"].values + error_radii = non_null_df["error_radius"].values + event_types = non_null_df["event_type"].values + force_retains = non_null_df["force_retain"].values + update_statuses = non_null_df["update_status"].values + last_updated_bys = non_null_df["last_updated_by"].values + + for i in range(total_non_null_pings): + segment_layer = layers_dict[segments[i]] + + if i != (total_non_null_pings - 1): + fl.PolyLine(locations=[clean_trace_geometry[i], clean_trace_geometry[i + 1]], + color="red", weight=3, opacity=1, tags=["clean"], ).add_to(segment_layer) - counter += 1 - icon = plugins.BeautifyIcon(icon_shape="circle", - number=index + 1, - border_color="blue", - background_color="transparent", + icon = plugins.BeautifyIcon(icon_shape="circle", number=i + 1, + border_color="blue", background_color="transparent", inner_icon_style="font-size:12px;") - clean_coordinates = [row["cleaned_latitude"], - row["cleaned_longitude"]] - - fl.Marker(clean_coordinates, + fl.Marker([clean_lats[i], clean_lngs[i]], icon=icon, - tooltip="Index: " + str(index + 1) + - "_ping_id: " + row["ping_id"], + tooltip="Index: " + str(i + 1) + "_ping_id: " + str(ping_ids[i]), tags=[""], - popup=create_general_popup({"ping_id": row["ping_id"], - "input_latitude": None if pd.isna(row["input_latitude"]) else row["input_latitude"], - "input_longitude": None if pd.isna(row["input_longitude"]) else row["input_longitude"], - "timestamp": int(row["timestamp"]), - "time_string": convert_unix_timestamp_to_human_readable(int((row["timestamp"])//1000)), - "error_radius": None if pd.isna(row["error_radius"]) else row["error_radius"], - "event_type": None if pd.isna(row["event_type"]) else row["event_type"], - "force_retain": row["force_retain"], - "cleaned_latitude": None if pd.isna(row["cleaned_latitude"]) else row["cleaned_latitude"], - "cleaned_longitude": None if pd.isna(row["cleaned_longitude"]) else row["cleaned_longitude"], - "update_status": row["update_status"], - "last_updated_by": row["last_updated_by"] - }) + popup=create_general_popup({"ping_id": ping_ids[i], + "input_latitude": None if pd.isna(input_lats[i]) else input_lats[i], + "input_longitude": None if pd.isna(input_lngs[i]) else input_lngs[i], + "timestamp": int(timestamps[i]), + "time_string": convert_unix_timestamp_to_human_readable(int(timestamps[i]//1000)), + "error_radius": None if pd.isna(error_radii[i]) else error_radii[i], + "event_type": None if pd.isna(event_types[i]) else event_types[i], + "force_retain": force_retains[i], + "cleaned_latitude": clean_lats[i], + "cleaned_longitude": clean_lngs[i], + "update_status": update_statuses[i], + "last_updated_by": last_updated_bys[i]}) ).add_to(segment_layer) fl.LayerControl().add_to(map_object) @@ -257,66 +238,56 @@ def _plot_before_or_after_operation(trace_df, layers_dict[segment_id] = raw_trace_segment_layer - for index, row in trace_df.iterrows(): - - if ((row["cleaned_latitude" + suffix] is None) or - (row["cleaned_longitude" + suffix] is None) or - (pd.isna(row["cleaned_latitude" + suffix])) or - (pd.isna(row["cleaned_longitude" + suffix]))): - continue - - raw_trace_geometry.append((row["cleaned_latitude" + suffix],row["cleaned_longitude" + suffix])) + # Build geometry from non-null pings using vectorized filter + lat_col = "cleaned_latitude" + suffix + lng_col = "cleaned_longitude" + suffix + non_null_mask = trace_df[lat_col].notna() & trace_df[lng_col].notna() + non_null_df = trace_df[non_null_mask].reset_index(drop=True) + c_lats = non_null_df[lat_col].values + c_lngs = non_null_df[lng_col].values + raw_trace_geometry = list(zip(c_lats, c_lngs)) total_non_null_pings = len(raw_trace_geometry) - counter = 0 - for index, row in trace_df.iterrows(): - - if ((row["cleaned_latitude" + suffix] is None) or - (row["cleaned_longitude" + suffix] is None) or - (pd.isna(row["cleaned_latitude" + suffix])) or - (pd.isna(row["cleaned_longitude" + suffix]))): - continue - - segment_id = row["trace_segment"] - segment_layer = layers_dict[segment_id] - - if (counter != (total_non_null_pings - 1)): - start = raw_trace_geometry[counter] - end = raw_trace_geometry[counter + 1] - fl.PolyLine(locations=[start, end], - color="red", - weight=3, - opacity=1, - tags=["raw"], + segments = non_null_df["trace_segment"].values + ping_ids = non_null_df["ping_id"].values + i_lats = non_null_df["input_latitude" + suffix].values + i_lngs = non_null_df["input_longitude" + suffix].values + timestamps = non_null_df["timestamp" + suffix].values + error_radii = non_null_df["error_radius" + suffix].values + event_types = non_null_df["event_type" + suffix].values + force_retains = non_null_df["force_retain" + suffix].values + update_statuses = non_null_df["update_status" + suffix].values + last_updated_bys = non_null_df["last_updated_by" + suffix].values + + for i in range(total_non_null_pings): + segment_layer = layers_dict[segments[i]] + + if i != (total_non_null_pings - 1): + fl.PolyLine(locations=[raw_trace_geometry[i], raw_trace_geometry[i + 1]], + color="red", weight=3, opacity=1, tags=["raw"], ).add_to(segment_layer) - counter += 1 - icon = plugins.BeautifyIcon(icon_shape="circle", - number=index + 1, - border_color="blue", - background_color="transparent", + icon = plugins.BeautifyIcon(icon_shape="circle", number=i + 1, + border_color="blue", background_color="transparent", inner_icon_style="font-size:12px;") - raw_coordinates = [row["cleaned_latitude" + suffix],row["cleaned_longitude" + suffix]] - - fl.Marker(raw_coordinates, + fl.Marker([c_lats[i], c_lngs[i]], icon=icon, - tooltip="Index: " + str(index + 1) + - "_ping_id: " + row["ping_id"], + tooltip="Index: " + str(i + 1) + "_ping_id: " + str(ping_ids[i]), tags=[""], - popup=create_general_popup({"ping_id": row["ping_id"], - "input_latitude": None if pd.isna(row["input_latitude" + suffix]) else row["input_latitude" + suffix], - "input_longitude": None if pd.isna(row["input_longitude" + suffix]) else row["input_longitude" + suffix], - "timestamp": int(row["timestamp" + suffix]), - "time_string": convert_unix_timestamp_to_human_readable(int((row["timestamp" + suffix])//1000)), - "error_radius": None if pd.isna(row["error_radius" + suffix]) else row["error_radius" + suffix], - "event_type": None if pd.isna(row["event_type" + suffix]) else row["event_type" + suffix], - "force_retain": row["force_retain" + suffix], - "cleaned_latitude": None if pd.isna(row["cleaned_latitude" + suffix]) else row["cleaned_latitude" + suffix], - "cleaned_longitude": None if pd.isna(row["cleaned_longitude" + suffix]) else row["cleaned_longitude" + suffix], - "update_status": row["update_status" + suffix], - "last_updated_by": row["last_updated_by" + suffix]}) + popup=create_general_popup({"ping_id": ping_ids[i], + "input_latitude": None if pd.isna(i_lats[i]) else i_lats[i], + "input_longitude": None if pd.isna(i_lngs[i]) else i_lngs[i], + "timestamp": int(timestamps[i]), + "time_string": convert_unix_timestamp_to_human_readable(int(timestamps[i]//1000)), + "error_radius": None if pd.isna(error_radii[i]) else error_radii[i], + "event_type": None if pd.isna(event_types[i]) else event_types[i], + "force_retain": force_retains[i], + "cleaned_latitude": c_lats[i], + "cleaned_longitude": c_lngs[i], + "update_status": update_statuses[i], + "last_updated_by": last_updated_bys[i]}) ).add_to(segment_layer) return map_object diff --git a/src/tracely/utils/utils.py b/src/tracely/utils/utils.py index 5d14f59..938e0db 100644 --- a/src/tracely/utils/utils.py +++ b/src/tracely/utils/utils.py @@ -2,10 +2,13 @@ import math from typing import Union import datetime +import numpy as np from haversine import haversine from .. import constants +_EARTH_RADIUS_M = 6371000 + def get_haversine_distance(lat_1: Union[int, float], lng_1: Union[int, float], @@ -25,16 +28,43 @@ def get_haversine_distance(lat_1: Union[int, float], """ try: - haversine_distance = haversine((lat_1, lng_1), - (lat_2, lng_2), + haversine_distance = haversine((lat_1, lng_1), + (lat_2, lng_2), unit="m") - + return round(haversine_distance, 2) - + except Exception: return None +def vectorized_haversine(lat1, lng1, lat2, lng2): + """ + Compute haversine distances between arrays of coordinates using vectorized numpy operations. + + Args: + lat1, lng1, lat2, lng2: numpy arrays of coordinates in decimal degrees. + + Returns: + numpy array: Distances in meters (rounded to 2 decimal places). NaN where any input is NaN. + """ + lat1 = np.asarray(lat1, dtype=np.float64) + lng1 = np.asarray(lng1, dtype=np.float64) + lat2 = np.asarray(lat2, dtype=np.float64) + lng2 = np.asarray(lng2, dtype=np.float64) + + lat1_r = np.radians(lat1) + lat2_r = np.radians(lat2) + dlat = np.radians(lat2 - lat1) + dlng = np.radians(lng2 - lng1) + + a = np.sin(dlat / 2) ** 2 + np.cos(lat1_r) * np.cos(lat2_r) * np.sin(dlng / 2) ** 2 + c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a)) + distances = np.round(_EARTH_RADIUS_M * c, 2) + + return distances + + def calculate_trace_distance(trace): """ This function gives the total distance calculated from a trace, using haversine method.