#!/usr/bin/env python3
'''remove oldest frame files that exceed disk usage limits'''
# Changes
#  - Determines total disk usages (in dk output) from Available + Free.
#    Testing on Ubuntu12, Debian8 showed that # of 1k blocks was wrong
#  - If a directory does not exist, it skips over it
#
#   -- Keith Thorne 2017-11-29
#   -- Keith Thorne 2024-09-26 update for Debian , modified percentages for small test stands
#   -- Keith Thorne 2024-10-08 Converted from Perl to Python3
#                              Allows environment variables to configure limits
#                              Allows environment variables to configure frame directories
#
#  ** Default behavior is still a 'dry run' with no deletions
#  Add command line parameter '--delete' to activate deletion

# imports
import datetime
import sys
import os
import shutil
import glob

nowtime = datetime.datetime.now()
print(nowtime.strftime("%Y-%m-%d %H:%M:%S"))

DRY_RUN = True
NUM_ARG = len(sys.argv)
if NUM_ARG > 1:
    FIRST_ARG = sys.argv[1]
    if FIRST_ARG == '--delete':
        DRY_RUN = False

if DRY_RUN:
    print("Dry run -  will not remove any files!!!")
    print("You need to rerun this with --delete argument to really delete frame files")
else:
    print("Active run - will delete files as needed")

# Settings for frame space to keep
# Override defaults with environment variables
#  -- defaults are for small test stand with smaller frame disk
#
#  Max % space for frame files DAQ_WIPER_MAX_KEEP
#  Max % of frame file space for full frames DAQ_WIPER_FULL_KEEP
#  Max % of frame file space for second trend frames DAQ_WIPER_SEC_KEEP
#  Max % of frame file space for minute trend frames DAQ_WIPER_MIN_KEEP

PERCENT_KEEP = 90.0
ext_max_keep = os.environ.get('DAQ_WIPER_MAX_KEEP')
if ext_max_keep is not None:
    try:
        ext_max_keep_pct = float(ext_max_keep)
        if 0 < ext_max_keep_pct < 100:
            PERCENT_KEEP =  ext_max_keep_pct
        else:
            print(f"DAQ_WIPER_MAX_KEEP of {ext_max_keep_pct} out of range - use default")
    except ValueError:
        print(f"DAQ_WIPER_MAX_KEEP of {ext_max_keep} is bad - use default")
print(f"Max % space for frame files: {PERCENT_KEEP:0.2f}")

# ######################################################
# Below keep values are percentages within $PERCENT_KEEP
# minus the size of raw minute trend data, which we do not delete
# This script deletes full, second and minute trend files
# to bring the file system usage to the specified values.

# Keep full frames under this percentage
FULL_FRAMES_PERCENT_KEEP = 75.0
ext_full_keep = os.environ.get('DAQ_WIPER_FULL_KEEP')
if ext_full_keep is not None:
    try:
        ext_full_keep_pct=float(ext_full_keep)
        if 0 < ext_full_keep_pct <= 100:
            FULL_FRAMES_PERCENT_KEEP = ext_full_keep_pct
        else:
            print(f"DAQ_WIPER_FULL_KEEP of {ext_full_keep_pct} out of range - use default")
    except ValueError:
        print(f"DAQ_WIPER_FULL_KEEP of {ext_full_keep} is bad - use default")
print(f"Max % fraction for frame files: {FULL_FRAMES_PERCENT_KEEP:0.2f}")

# Keep second trend frames under this percentage
SECOND_FRAMES_PERCENT_KEEP = 24.5
ext_second_keep = os.environ.get('DAQ_WIPER_SECOND_KEEP')
if ext_second_keep is not None:
    try:
        ext_second_keep_pct=float(ext_second_keep)
        if 0 <= ext_second_keep_pct < 100:
            SECOND_FRAMES_PERCENT_KEEP =  ext_second_keep_pct
        else:
            print(f"DAQ_WIPER_SECOND_KEEP of {ext_second_keep_pct} out of range - use default")
    except ValueError:
        print(f"DAQ_WIPER_SECOND_KEEP of {ext_second_keep} is bad - use default")
print(f"Max % fraction for second trend frame files: {SECOND_FRAMES_PERCENT_KEEP:0.2f}")

# Keep minute trend frames under this percentage
MINUTE_FRAMES_PERCENT_KEEP = 0.5
ext_minute_keep = os.environ.get('DAQ_WIPER_MINUTE_KEEP')
if ext_minute_keep is not None:
    try:
        ext_minute_keep_pct=float(ext_minute_keep)
        if 0 <= ext_minute_keep_pct < 100:
            MINUTE_FRAMES_PERCENT_KEEP =  ext_minute_keep_pct
        else:
            print(f"DAQ_WIPER_MINUTE_KEEP of {ext_minute_keep_pct} out of range - use default")
    except ValueError:
        print(f"DAQ_WIPER_MINUTE_KEEP of {ext_minute_keep} is bad - use default")
print(f"Max % fraction for minute trend frame files: {MINUTE_FRAMES_PERCENT_KEEP:0.2f}")

# we can also configure the locations of the frames disk and frame sub-directories
# DAQ_FRAMES_DIR
# DAQ_FULL_FRAMES_SUBDIR
# DAQ_SECOND_FRAMES_SUBDIR
# DAQ_MINUTE_FRAMES_SUBDIR
# DAQ_RAW_TRENDS_SUBDIR

# Frame file system mount point
FRAMES_DIR = "/frames"
ext_frames_dir = os.environ.get('DAQ_FRAMES_DIR')
if ext_frames_dir is not None:
    if os.path.isdir(ext_frames_dir):
        FRAMES_DIR = ext_frames_dir

# Full frame directory
FULL_FRAMES_DIR = FRAMES_DIR + "/full"
ext_full_subdir = os.environ.get('DAQ_FULL_FRAMES_SUBDIR')
if ext_full_subdir is not None:
    ext_full_dir = FRAMES_DIR + "/" + ext_full_subdir
    if os.path.isdir(ext_full_dir):
        FULL_FRAMES_DIR = ext_full_dir

# Second trend frame directory
SECOND_TRENDS_FRAMES_DIR = FRAMES_DIR + "/trend/second"
ext_second_subdir = os.environ.get('DAQ_SECOND_FRAMES_SUBDIR')
if ext_second_subdir is not None:
    ext_second_dir = FRAMES_DIR + "/" + ext_second_subdir
    if os.path.isdir(ext_second_dir):
        SECOND_TRENDS_FRAMES_DIR = ext_second_dir

# Minute trend frame directory
MINUTE_TRENDS_FRAMES_DIR = FRAMES_DIR + "/trend/minute"
ext_minute_subdir = os.environ.get('DAQ_MINUTE_FRAMES_SUBDIR')
if ext_minute_subdir is not None:
    ext_minute_dir = FRAMES_DIR + "/" + ext_minute_subdir
    if os.path.isdir(ext_minute_dir):
        MINUTE_TRENDS_FRAMES_DIR = ext_minute_dir

# Raw minute trend file directory (may not exist)
RAW_MINUTE_TRENDS_FRAMES_DIR = FRAMES_DIR + "/trend/minute_raw"
ext_raw_subdir = os.environ.get('DAQ_RAW_TRENDS_SUBDIR')
if ext_raw_subdir is not None:
    ext_raw_dir = FRAMES_DIR + "/" + ext_raw_subdir
    if os.path.isdir(ext_raw_dir):
        RAW_MINUTE_TRENDS_FRAMES_DIR = ext_raw_dir

# create dictionary of file/directory sizes
du = {}

# number of bytes in kilobyte
KB = 1024

# it is amazing how many lines of Python are needed to replace 'du -sk'
def ddu(path='.'):
    '''return disk usage of file/directory'''
    if os.path.exists(path):
        total = 0
        with os.scandir(path) as item:
            for entry in item:
                if entry.is_file():
                    total += entry.stat().st_size
                elif entry.is_dir():
                    total += ddu(entry.path)
        return total
    return 0

# Determine usage for each directory
du[FULL_FRAMES_DIR] = ddu(FULL_FRAMES_DIR) / KB
du[SECOND_TRENDS_FRAMES_DIR] = ddu(SECOND_TRENDS_FRAMES_DIR) / KB
du[MINUTE_TRENDS_FRAMES_DIR] = ddu(MINUTE_TRENDS_FRAMES_DIR) / KB
du[RAW_MINUTE_TRENDS_FRAMES_DIR] = ddu(RAW_MINUTE_TRENDS_FRAMES_DIR) /KB

# show disk usage summary
print("Directory disk usage:")
combined_kb = 0
for direct, dir_size in du.items():
    combined_kb += dir_size
    print(f" {direct} {dir_size:0.2f} KB")

combined_mb = combined_kb / KB
combined_gb = combined_mb / KB
print(f"Combined {combined_kb:0.2f} KB or {combined_mb:0.2f} MB or {combined_gb:0.2f} GB")

usage_kbytes = combined_kb

# file system overall size
df_stat = shutil.disk_usage(FRAMES_DIR)
df_kbytes = df_stat.total / KB
df_percent_kbytes = df_kbytes / 100
df_used_kbytes = df_stat.used / KB
df_free_kbytes = df_stat.free / KB
percent_usage = df_used_kbytes / df_percent_kbytes
print(f"frames_dir size {df_kbytes:0.2f} KB at {percent_usage:0.2f} %")

if percent_usage <= PERCENT_KEEP:
    print(f"frames_dir is below keep value of {PERCENT_KEEP:0.2f} %")
    print("Will not delete any files")
    print(f"df reported usage {percent_usage:0.2f} %")
    exit(0)
else:
    print(f"frames_dir is above keep value of {PERCENT_KEEP:0.2f} %")

frame_area_size =  ( df_percent_kbytes * PERCENT_KEEP ) - du[RAW_MINUTE_TRENDS_FRAMES_DIR]
print(f"Frame area size is {frame_area_size:0.2f} KB")
frame_area_percent = frame_area_size / 100.0

if os.path.exists(FULL_FRAMES_DIR):
    full_frames_keep = FULL_FRAMES_PERCENT_KEEP * frame_area_percent
    print(f"FULL_FRAMES_DIR size {du[FULL_FRAMES_DIR]:0.2f} KB keep {full_frames_keep:0.2f} KB")
    DO_FULL = du[FULL_FRAMES_DIR] > full_frames_keep
else:
    DO_FULL = False

if os.path.exists(SECOND_TRENDS_FRAMES_DIR):
    second_frames_keep = SECOND_FRAMES_PERCENT_KEEP * frame_area_percent
    print(f"SECOND_TRENDS_FRAMES_DIR size {du[SECOND_TRENDS_FRAMES_DIR]:0.2f} KB \
keep {second_frames_keep:0.2f} KB")
    DO_SEC = du[SECOND_TRENDS_FRAMES_DIR] > second_frames_keep
else:
    DO_SEC = False

if os.path.exists(MINUTE_TRENDS_FRAMES_DIR):
    minute_frames_keep = MINUTE_FRAMES_PERCENT_KEEP * frame_area_percent
    print(f"MINUTE_TRENDS_FRAMES_DIR size {du[MINUTE_TRENDS_FRAMES_DIR]:0.2f} KB \
keep {minute_frames_keep:0.2f} KB")
    DO_MIN = du[MINUTE_TRENDS_FRAMES_DIR] > minute_frames_keep
else:
    DO_MIN = False

def delete_frames (dir_path,ktofree):
    ''' Delete frame files in directory path to free input Kbytes of space'''
    # This one reads file names in $dir/*/*.gwf sorts them by GPS time
    # and progressively deletes them up to $ktofree limit
    # Limit search to GW files
    gwf_search = dir_path + '/*/*.gwf'
    gw_files = sorted(glob.glob(gwf_search))
    fnum = len(gw_files)
    dacc = 0
    dnum = 0
    for file in gw_files:
        file_size = os.path.getsize(file) / 1024
        dacc += file_size
        if dacc >= ktofree:
            break
        dnum += 1
    # Delete file here
        if DRY_RUN is False:
            os.unlink(file)
            print(f" Unlink {file}")
        else:
            print(f" Dry run - we would delete {file}")

    print(f"Overall {fnum}; deleted {dnum}")

# See what we need to remove in full frames
if DO_FULL is True:
    k_need_to_free = du[FULL_FRAMES_DIR] -  full_frames_keep
    print(f"Deleting some full frames to free {k_need_to_free:0.2f} kB")
    delete_frames(FULL_FRAMES_DIR,k_need_to_free)

# See if anythings needs to go in second trends
if DO_SEC is True:
    k_need_to_free = du[SECOND_TRENDS_FRAMES_DIR] - second_frames_keep
    print(f"Deleting some second trend frames to free {k_need_to_free:0.2f}")
    delete_frames(SECOND_TRENDS_FRAMES_DIR,k_need_to_free)

if DO_MIN is True:
    k_need_to_free =  du[MINUTE_TRENDS_FRAMES_DIR] - minute_frames_keep
    print(f"Deleting some minute trend frames to free {k_need_to_free:0.2f}")
    delete_frames(MINUTE_TRENDS_FRAMES_DIR, k_need_to_free)
