import boto3
import subprocess
from datetime import datetime

# Connect to MinIO
s3 = boto3.client(
    's3',
    endpoint_url='http://localhost:9002',
    aws_access_key_id='admin',
    aws_secret_access_key='admin123456'
)

# List all files in raw-logs bucket
response = s3.list_objects_v2(Bucket='raw-logs', Prefix='logs/')

if 'Contents' not in response:
    print("No files found in MinIO")
    exit()

print(f"Found {len(response['Contents'])} files in MinIO, starting transfer to HDFS...")

for obj in response['Contents']:
    key = obj['Key']

    # Download file from MinIO
    file_obj = s3.get_object(Bucket='raw-logs', Key=key)
    content  = file_obj['Body'].read()

    # Save temporarily on local machine
    tmp_path = "/tmp/hdfs_upload.json"
    with open(tmp_path, 'wb') as f:
        f.write(content)

    # Create HDFS directory first (important!)
    hdfs_dir = "/ids/" + "/".join(key.split("/")[:-1])
    subprocess.run([
        "docker", "exec", "namenode",
        "hdfs", "dfs", "-mkdir", "-p", hdfs_dir
    ])

    # Copy file into namenode container
    subprocess.run([
        "docker", "cp", tmp_path, "namenode:/tmp/hdfs_upload.json"
    ])

    # Upload from namenode container to HDFS
    hdfs_path = f"/ids/{key}"
    result = subprocess.run([
        "docker", "exec", "namenode",
        "hdfs", "dfs", "-put", "-f",
        "/tmp/hdfs_upload.json", hdfs_path
    ], capture_output=True, text=True)

    if result.returncode == 0:
        print(f"Moved to HDFS: {hdfs_path}")
    else:
        print(f"Failed: {hdfs_path} — {result.stderr.strip()}")

print("Done! All files transferred to HDFS.")
