Commit 9c8e7417 authored by Tilmann Sager's avatar Tilmann Sager
Browse files

Added saving results

parent 3f3ca9cf
......@@ -3,13 +3,11 @@ name = MODTBGA
bands = 31, 32
11um = 31
12um = 32
calculate_radiance = 0
[data]
root = ../contraildetect/data/
hdf = ../contraildetect/data/input/hdf
flights = ../contraildetect/data/input/flights
processed = ../contraildetect/processed
tmp = ../contraildetect/data/tmp
hdf = ../contraildetect/data/hdf
flights = ../contraildetect/data/flights
output = ../contraildetect/data/output
[extraction]
......
......@@ -24,10 +24,6 @@ def product_name():
return cp.get(prod_key, 'name')
def product_dir():
return os.path.join(hdf_input(), product_name())
def bands():
return [int(band.strip()) for band in cp.get(prod_key, 'bands').split(',')]
......@@ -47,30 +43,23 @@ DIRECTORIES
data_key = 'data'
def hdf_input():
return cp.get(data_key, 'hdf')
def product_dir():
return os.path.join(cp.get(data_key, 'hdf'), product_name())
def flight_input():
return cp.get(data_key, 'flights')
def processed():
proc_dir = os.path.join(cp.get(data_key, 'processed'), _today())
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
return proc_dir
def processed_flights():
proc_flight_dir = os.path.join(processed(), 'flights')
proc_flight_dir = os.path.join(flight_input(), 'processed')
if not os.path.exists(proc_flight_dir):
os.makedirs(proc_flight_dir)
return proc_flight_dir
def processed_img():
proc_img_dir = os.path.join(processed(), 'img')
def output_img():
proc_img_dir = os.path.join(output_dir(), 'img')
if not os.path.exists(proc_img_dir):
os.makedirs(proc_img_dir)
return proc_img_dir
......
import os
from glob import glob
from tqdm import tqdm
import config
import pipeline
from stages.flight_filtering import filter_flights
from stages.prepare import create_job
from constants import columns_to_save
from stages import prepare, transformation, detection, flight_filtering, segmentation, assembling
from util import check_and_repair_csv
import pandas as pd
tqdm.pandas()
"""
INPUT
"""
# input_files = glob(config.product_dir() + '/*.hdf')
input_files = [os.path.join(config.product_dir(), 'MODTBGA.A2012094.h15v03.006.2015240000438.hdf')]
input_files = glob(config.product_dir() + '/*.hdf')
# input_files = [os.path.join(config.product_dir(), 'MODTBGA.A2012094.h15v03.006.2015240000438.hdf')]
# TODO: collecting system information (cpu cores, current time)
# TODO: multiprocessing
# TODO: implement opt-out/in stages
# TODO: improve logging
job_df = create_job(input_files)
print('Creating job')
job_df = prepare.create_job(input_files)
# TODO: print status
"""
CHECK FLIGHT CSV
"""
print('Checking flight files')
check_and_repair_csv(job_df)
# TODO: print status
......@@ -32,14 +37,36 @@ check_and_repair_csv(job_df)
"""
FILTER FLIGHT CSV
"""
job_df = filter_flights(job_df)
print('Filter flights')
job_df = job_df.progress_apply(flight_filtering.filter_flights, axis=1)
"""
RUN PIPELINE
TRANSFORMATION
"""
job_df = job_df.apply(pipeline.run, axis=1)
print('Running CDA/thresholding')
job_df = job_df.progress_apply(transformation.cda_preprocess, axis=1)
job_df = job_df.progress_apply(transformation.threshold, axis=1)
"""
RESULTS
DETECTION
"""
print('Running detection')
job_df = job_df.progress_apply(detection.probabilistic_hough, axis=1)
"""
SEGMENTATION
"""
print('Running segmentation')
job_df = job_df.progress_apply(segmentation.run, axis=1)
"""
ASSEMBLING INFORMATION
"""
print('Assembling information')
job_df = job_df.progress_apply(assembling.collect, axis=1)
"""
RESULTS
"""
print('Saving results')
job_df.to_csv(os.path.join(config.output_dir(), 'output.csv'), columns=columns_to_save, index=False)
......@@ -50,7 +50,7 @@ def _filter_by_time_and_position(day_segment_df: pd.DataFrame, south: float, nor
return computed_df
def _filter_flights(row: pd.Series):
def filter_flights(row: pd.Series):
if row[col.flight_processed] is not None:
flight_prc = pd_read_csv(row[col.flight_processed], DTYPES)
row[col.flight_count] = len(flight_prc['FLIGHT_ID'].unique().tolist())
......@@ -74,7 +74,3 @@ def _filter_flights(row: pd.Series):
del flights_fil
return row
def filter_flights(job_df: pd.DataFrame):
return job_df.apply(_filter_flights, axis=1)
......@@ -50,7 +50,7 @@ def create_row(filepath: str):
}
def create_job(filepath_list):
def create_job(filepath_list) -> pd.DataFrame:
job_df = pd.DataFrame()
for filepath in filepath_list:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment