다운로드한 파일이 파일 이름의 첫 글자를 포함하는 하위 폴더로 나누어지도록 매우 큰 URL 목록을 다운로드하려면 어떻게 해야 합니까?

어쩌면 다음과 같은 것일 수도 있습니다.

awk -F/ '{print substr($NF, 1, 1), $0}' urls.txt |
  xargs -L1 bash -c 'mkdir -p -- "$0" && curl -sSF -O --output-dir "$0" "$1"'

각 줄 앞에 awk파일 이름의 첫 번째 문자를 추가하고 해당 문자를 사용하여 curl명령에서 출력 디렉터리를 선택합니다. -PGNU 구현 옵션을 사용하여 xargs여러 추출을 병렬로 실행할 수 있습니다 .

URL에는 공백, 따옴표 또는 백슬래시가 포함되어 있지 않은 것으로 간주되지만 URL에는 URI 인코딩 이외의 다른 내용이 포함되어서는 안 됩니다( curl직접 처리하고 URI 인코딩을 수행할 수 있더라도).

예제 입력이 주어지면 위 명령을 실행하면 다음이 생성됩니다.

.
├── 0
│   └── 0wd.pdf
├── a
│   ├── asz.pdf
│   └── axz.pdf
└── b
    ├── b00.pdf
    └── bb0.pdf

ChatGPT는 Python에서 일부 작동 코드를 제공합니다(Python 3.11에서 작동하는 것을 확인했습니다).

import os import requests

def download_files_with_subfolders(url_file):
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            # Download the file
            response = requests.get(url)
            if response.status_code == 200:
                file_path = os.path.join(subfolder, filename)
                with open(file_path, 'wb') as file:
                    file.write(response.content)
                print(f"Downloaded: {url} -> {file_path}")
            else:
                print(f"Failed to download: {url} (Status code: {response.status_code})")

if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file) 

포함하다 somefile.txt:

http://mydomain.com/0wd.pdf
http://mydomain.com/asz.pdf
http://mydomain.com/axz.pdf
http://mydomain.com/b00.pdf
http://mydomain.com/bb0.pdf
etc.

---

고급 변형:

1. 응답 헤더에 마지막 수정 날짜를 유지합니다(대부분 ChatGPT의 코드이기도 함).

import requests
import os
from datetime import datetime

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file):
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            file_path = os.path.join(subfolder, filename)
            download_file(url, file_path)

if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file)

2. 멀티스레드 다운로드:

import requests
import os
from datetime import datetime

from multiprocessing.dummy import Pool as ThreadPool

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file, num_threads=4):
    download_arguments = []
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            file_path = os.path.join(subfolder, filename)
            download_arguments.append((url, file_path))

    pool = ThreadPool(num_threads)
    results = pool.starmap(download_file, download_arguments)


if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file, num_threads=10)

3. 첫 번째 문자에 대한 폴더와 두 번째 문자에 대한 하위 폴더를 만듭니다. 예를 들어,:

0/w/0wd.pdf
a/s/asz.pdf
a/x/axz.pdf
b/0/b00.pdf
b/b/bb0.pdf
etc.

암호:

import requests
import os
from datetime import datetime

from multiprocessing.dummy import Pool as ThreadPool

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file, num_threads=4):
    download_arguments = []
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]
            second_letter = filename[1]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)
            subsubfolder = os.path.join(first_letter, second_letter)
            os.makedirs(subsubfolder, exist_ok=True)

            file_path = os.path.join(subsubfolder, filename)
            download_arguments.append((url, file_path))

    pool = ThreadPool(num_threads)
    results = pool.starmap(download_file, download_arguments)


if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file, num_threads=10)