1
#!/bin/bash
2

3
LOCAL_SOURCE_DIR="/data/local_files"  # 本地源文件目录
4
HDFS_TARGET_DIR="/user/hadoop/batch_uploads"  # hdfs目标目录
5
FILE_FILTER="*.txt"  # 文件过滤模式
6

7

8
LOG_FILE="./hdfs_upload_$(date +%Y%m%d).log"  # 日志文件路径
9

10
echo "=== 批量上传开始于 $(date) ===" >> $LOG_FILE
11

12
if [ ! -d "LOCAL_SOURCE_DIR" ]; then
13
    echo "error: 本地目录 $LOCAL_SCOURCE_DIR 不存在！" | tee -a $LOG_FILE
14
    exit 1
15
fi
16

17
if ! hdfs dfs -test -d "$HDFS_TARGET_DIR"; then
18
    echo "HDFS目录 $HDFS_TARGET_DIR 不存在，正在创建..." | tee -a $LOG_FILE
19
    hdfs dfs -mkdir -p "$HDFS_TARGET_DIR";
20
    if [ $? -ne 0 ]; then
21
        echo "error: Failed to create HDFS directory." | tee -a $LOG_FILE
22
        exit 1
23
    fi
24
fi
25

26
echo "开始上传文件（过滤条件：$FILE_FILTER）..." | tee -a $LOG_FILE
27

28
file_count=0
29
success_count=0
30
fail_count=0
31

32
for local_file in "$LOCAL_SCOURCE_DIR"/$FILE_FILTER; do
33
    if [ -f "$local_file" ]; then
34
        ((file_count++))
35

36
        filename=$(base "local_file")
37
        echo "正在上传：$filename" | tee -a $LOG_FILE
38

39
        hdfs dfs -put "$local_file" "$HDFS_TARGET_DIR/$filename"
40

41
        if [ $? -eq 0 ]; then
42
            ((success_count++))
43
            echo "Success: ${filename} uploaded succesfully" | tee -a $LOG_FILE
44
        else
45
            ((fail_count++))
46
            echo "Failure: Failed to upload ${filename}" | tee -a $LOG_FILE
47
        fi
48
    fi
49
done
50

51
echo "=== 批量上传结束于 $(date) ===" | tee -a $LOG_FILE
52
## ${变量名}  $(命令)
53
echo "Total files: ${file_count}, succeed: ${success_count},failed: $fail_count" | tee -a $LOG_FILE
54
echo "HDFS target directory: $(hdfs dfs -ls ${HDFS_TARGET_DIR})" | tee -a $LOG_FILE
55
echo "Log file: ${LOG_FILE}" | tee -a $LOG_FILE

1
#!/bin/bash
2

3

4
HDFS_SOURCE_DIR="/root/hadoop/batch_uploads"
5
LOCAL_TARGET_DIR="/data/downloaded_files"
6
LOG_FILE="./hdfs_download_$(date +%Y%m%d).log"
7

8
echo "=== Batch download started at $(date) ===" >> $LOG_FILE
9

10
mkdir -p "$LOCAL_TARGET_DIR"
11

12
if [ $? -ne 0 ]; then
13
    echo "error: failed to create local directory: ${LOCAL_TARGET_DOR}" | tee -a $LOG_FILE
14
    exit 1
15
fi
16

17
if ! hdfs dfs -test -d "$HDFS_SOURCE_DIR"; then
18
    echo "error: HDFS directory does not exist: ${HDFS_SOURCE_DIR}" | tee -a $LOG_FILE
19
    exit 1
20
fi
21

22
echo "Downloading files..." | tee -a $LOG_FILE
23

24
file_count=0
25
success_count=0
26
fail_count=0
27
skip_count=0
28

29
while read hdfs_file; do
30
    filename=$(basename "$hdfs_file")
31
    local_file="$LOCAL_TARGET_DIR/$filename"
32

33
    if [ -f "local_file" ]; then
34
        ((skip_count++))
35
        echo "skipped: ${filename} already exists locally" | tee -a $LOG_FILE
36
        continue
37
    fi
38

39
    if [ -z "$hdfs_file" ];then
40
        ((skip_count++))
41
        echo "skipped: empty HDFS file detected: ${hdfs_file}" | tee -a $LOG_FILE
42
        continue
43
    fi
44
    ((file_count++))
45

46
    echo "downloading: ${filename}" | tee -a $LOG_FILE
47
    hdfs dfs -get "$hdfs_file" "$local_file"
48

49
    if [ $? -eq 0 ]; then
50
        ((success_count++))
51
        echo "Success: ${filename} downloaded successful" | tee -a $LOG_FILE
52
    else
53
        ((fail_count++))
54
        echo "Failure: ${filename} Failed to download" | tee -a $LOG_FILE
55
    fi
56
done < <(hdfs dfs -ls "$HDFS_SOURCE_DIR" | grep -v "^d" | awk '{print $8}')
57

58
echo "=== batch download ended at $(date) ===" | tee -a $LOG_FILE
59
echo "total: ${file_count},success: ${success_count},failure:${fail_count},skipped:${skip_count}" | tee -a $LOG_FILE
60
echo "log file: $LOG_FILE" | tee -a $LOG_FILE

1
#!/bin/bash
2

3
HDFS_DIR="/user/root/batch_uploads"
4
RETENTION_DAYS=7
5
LOG_FILE="cleanup_log.txt"
6

7
echo "starting cleanup at $(date)" | tee -a $LOG_FILE
8

9
hdfs dfs -ls $HDFS_DIR | grep -v "^d" | awk -v days=$RETENTION_DAYS '{
10
    date_str = $6 " " $7
11
    file = $8
12

13
    "date -d \"" date_str "\" +%s" | getline cutoff_ts
14

15
    if(file_ts < cutoff_ts) {
16

17
        print "Delete old file: " file
18
        system("hdfs dfs -rm " file " >> '${LOG_FILE}' 2>&1)
19

20
    }
21
}'
22

23
echo "Cleanup completed at $(date)" | tee -a $LOG_FILE

1
#!/bin/bash
2

3
LOCAL_SOURCE_DIR="/data/local_files"
4

5
if [ ! -d "$LOCAL_SOURCE_DIR" ]; then
6
    echo "warn: 本地目录 $LOCAL_SOURCE_DIR 不存在！"
7
    mkdir -p "LOCAL_SOURCE_DIR"
8
fi
9

10
echo "HDFS commend line test" > local_test.txt
11
mkdir -p local_data
12
for i in {1..5};do
13
    echo "Test file $i" > "${LOCAL_SOURCE_DIR}/file${i}.txt";
14
done