Python 多线程(多任务)下载假死求助

abpyu 发表于 2024-3-27 22:58

以下代码反复在AI上修改无数次
总是遇到各种问题
目前的问题是，只能下载前面五个任务，之后就假死不动了

例如txt文档内的连接是：
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-1x-ov.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-1x-ox.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-2x-ov.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-2x-ox.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-4x-ov.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-256x352-4x-ox.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-288x288-1x-ov.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-288x288-1x-ox.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-288x288-2x-ov.tz
http://veai-models.topazlabs.com/aaa-v10-fnet-fp16-288x288-2x-ox.tz

需要按照换行符拆分url，每次下载5个连接，直至所有任务下载完成，自己结束进程关闭，或者是在状态栏提醒，所有任务完成

import os
import tkinter as tk
from tkinter import filedialog, messagebox
import requests
from queue import Queue
from threading import Thread

# 定义全局变量
txt_path = ""
download_path = "C:\\ProgramData\\Topaz Labs LLC\\Topaz Photo AI\\models\\"
urls = []
download_queue = Queue()
thread_count = 5# 默认线程数
root = tk.Tk()

# 确保下载目录存在
def ensure_download_path():
global download_path
if not os.path.exists(download_path):
   os.makedirs(download_path)

# 更新状态栏
def update_status(text):
status_label.config(text=text)

# 选择TXT文档
def select_txt_file():
global txt_path, urls
file_path = filedialog.askopenfilename()
if file_path:
   with open(file_path, 'r') as file:
         urls =
   txt_path = file_path
   txt_txt_path.delete(0, tk.END)
   txt_txt_path.insert(0, txt_path)
   update_status(f"已载入 {len(urls)} 个URL。")

# 浏览保存文件夹
def select_download_path():
global download_path
folder_path = filedialog.askdirectory()
if folder_path:
   download_path = folder_path
   txt_download_path.delete(0, tk.END)
   txt_download_path.insert(0, download_path)
   update_status(f"已选择保存路径: {download_path}")

# 下载文件
def download_file(url, file_name):
file_path = os.path.join(download_path, file_name)
try:
   response = requests.get(url, stream=True)
   if response.status_code == 200:
         with open(file_path, 'wb') as file:
            for chunk in response.iter_content(chunk_size=8192):
               if chunk:
                     file.write(chunk)
   update_status(f'下载完成: {file_name}')
except Exception as e:
   update_status(f'下载错误: {file_name} (错误信息: {e})')

# 下载文件的线程函数
def thread_download_files():
while True:
   if not download_queue.empty():
         try:
            url, file_name = download_queue.get_nowait()
            download_file(url, file_name)
         except Exception as e:
            update_status(f'下载错误: {file_name} (错误信息: {e})')
         finally:
            download_queue.task_done()
   else:
         # 如果队列为空，等待一段时间再次检查
         # 如果等待一定时间后队列仍然为空，则退出线程
         time.sleep(1)
         break

# 开始下载
def start_download():
global urls, download_queue
if not urls:
   messagebox.showerror("错误", "没有可下载的URL。")
   return
if not os.path.exists(download_path):
   ensure_download_path()

# 重置状态栏
update_status("正在下载...")

# 清空队列并添加新的下载任务
download_queue = Queue()
for url in urls:
   file_name = url.split("/")[-1]
   download_queue.put((url, file_name))
   update_status(f"添加下载任务: {file_name}")

# 创建下载任务完成标志
download_complete =

# 创建并启动下载线程
threads = []
for _ in range(thread_count):# 创建线程处理所有任务
   thread = Thread(target=thread_download_files)
   thread.start()
   threads.append(thread)

# 等待所有下载线程完成
for thread in threads:
   thread.join()

# 所有任务完成后更新状态栏
download_complete = True
root.after(0, lambda: update_status("所有任务完成"))
root.after(0, root.quit)

# 创建GUI窗口
root.title("Topaz模型辅助下载器")

# 创建状态栏
status_frame = tk.Frame(root, bd=2, relief=tk.SUNKEN)
status_label = tk.Label(status_frame, text="就绪。", anchor="w")
status_label.pack(side="left", fill="x", padx=5, pady=5)
status_frame.pack(side="bottom", fill="x")

# 创建TXT路径相关控件
txt_path_frame = tk.Frame(root)
lbl_txt_path = tk.Label(txt_path_frame, text="TXT路径:")
lbl_txt_path.pack(side="left", padx=5, pady=5, anchor="w")
txt_txt_path = tk.Entry(txt_path_frame, width=50)
txt_txt_path.insert(0, txt_path)
txt_txt_path.pack(side="left", padx=5, pady=5, fill="x", expand=True)
btn_select_txt = tk.Button(txt_path_frame, text="浏览TXT", command=select_txt_file)
btn_select_txt.pack(side="left", padx=5, pady=5)

# 创建保存路径相关控件
download_path_frame = tk.Frame(root)
lbl_download_path = tk.Label(download_path_frame, text="保存路径:")
lbl_download_path.pack(side="left", padx=5, pady=5, anchor="w")
txt_download_path = tk.Entry(download_path_frame, width=50)
txt_download_path.insert(0, download_path)
txt_download_path.pack(side="left", padx=5, pady=5, fill="x", expand=True)
btn_select_download_path = tk.Button(download_path_frame, text="选择保存路径", command=select_download_path)
btn_select_download_path.pack(side="left", padx=5, pady=5)
btn_start_download = tk.Button(download_path_frame, text="开始下载", command=start_download)
btn_start_download.pack(side="left", padx=5, pady=5)

# 打包TXT路径相关控件
txt_path_frame.pack(side="top", fill="x", padx=5, pady=5)

# 打包保存路径相关控件
download_path_frame.pack(side="top", fill="x", padx=5, pady=5)

# 运行GUI
root.mainloop()

爱飞的猫 发表于 2024-3-28 05:43

本帖最后由爱飞的猫于 2024-3-28 05:46 编辑

推荐使用代码框来展示代码哦！

语言可以选择 `Python`。

参考：

- [【公告】发帖代码插入以及添加链接教程（有福利）](https://www.52pojie.cn/thread-713042-1-1.html)
- (https://www.52pojie.cn/thread-717627-1-1.html)

---

假死是因为你的主线程在等所有子线程执行结束。

首先把建立线程并等待结束的部分挪出来：

```py
def download_in_thread():
global download_queue

# 清空队列并添加新的下载任务
# ... 省略旧代码 ...

# 创建并启动下载线程
threads = []
for _ in range(thread_count):# 创建线程处理所有任务
   thread = Thread(target=thread_download_files)
   thread.start()
   threads.append(thread)

# 等待所有下载线程完成
for thread in threads:
   thread.join()

# 所有任务完成后更新状态栏
download_complete = True
root.after(0, lambda: update_status("所有任务完成"))
# 退出??
# root.after(0, root.quit)
```

在 `start_download` 建立新的线程，不等它结束：

```py
def start_download():
# ... 省略旧代码 ...

# 重置状态栏
update_status("正在下载...")
Thread(target=download_in_thread).start()
```

然后参考下 `Queue` 的文档，改了下获取数据和下载部分：

```py
# 下载文件的线程函数
def thread_download_files():
from queue import Empty

while True:
   try:
         url, file_name = download_queue.get(timeout=1)
   except Empty:
         break# 没有了

   try:
         download_file(url, file_name)
   except Exception as e:
         update_status(f'下载错误: {file_name} (错误信息: {e})')
   finally:
         download_queue.task_done()
```

> 目前的问题是，只能下载前面五个任务，之后就假死不动了

可能是还在下载，没下完。我把线程数量改成 1 然后构建了一些小文件的地址，是可以依次下载并正常完成任务。

调味包 发表于 2024-3-28 09:27

添加进度条，查看实时进度么。我目前就是为了看进度到那一步了{:1_901:}，或者是在添加一个时间监控下当前运行了多长时间，如果超出时间限制就报错退出

zerofire 发表于 2024-3-28 09:46

之前也遇到过类似的情况，而且是随机被卡死，停止下载，程序无响应，由于是抓取的别的网站内容，估计是不是触发了网站防抓取机制，当时还做了随机模拟抓取，但是也不行。

abpyu 发表于 2024-3-28 09:49

调味包发表于 2024-3-28 09:27
添加进度条，查看实时进度么。我目前就是为了看进度到那一步了，或者是在添加一个时间监控下当前 ...

改好的
确实是想加进度条的，不会加

import os
import tkinter as tk
from tkinter import filedialog, messagebox
import requests
from queue import Queue, Empty
from threading import Thread

# 定义全局变量
txt_path = ""
download_path = "C:\\ProgramData\\Topaz Labs LLC\\Topaz Photo AI\\models\\"
urls = []
download_queue = Queue()
thread_count = 5# 默认线程数
root = tk.Tk()

# 确保下载目录存在
def ensure_download_path():
global download_path
if not os.path.exists(download_path):
   os.makedirs(download_path)

# 更新状态栏
def update_status(text):
status_label.config(text=text)

# 选择TXT文档
def select_txt_file():
global txt_path, urls
file_path = filedialog.askopenfilename()
if file_path:
   with open(file_path, 'r') as file:
         urls =
   txt_path = file_path
   txt_txt_path.delete(0, tk.END)
   txt_txt_path.insert(0, txt_path)
   update_status(f"已载入 {len(urls)} 个URL。")

# 浏览保存文件夹
def select_download_path():
global download_path
folder_path = filedialog.askdirectory()
if folder_path:
   download_path = folder_path
   txt_download_path.delete(0, tk.END)
   txt_download_path.insert(0, download_path)
   update_status(f"已选择保存路径: {download_path}")

# 下载文件
def download_file(url, file_name):
file_path = os.path.join(download_path, file_name)
try:
   response = requests.get(url, stream=True)
   if response.status_code == 200:
         with open(file_path, 'wb') as file:
            for chunk in response.iter_content(chunk_size=8192):
               if chunk:
                     file.write(chunk)
   update_status(f'下载完成: {file_name}')
except Exception as e:
   update_status(f'下载错误: {file_name} (错误信息: {e})')

# 下载文件的线程函数
def thread_download_files():
while True:
   try:
         url, file_name = download_queue.get(timeout=1)
   except Empty:
         break# 队列为空，退出线程

   try:
         download_file(url, file_name)
   except Exception as e:
         update_status(f'下载错误: {file_name} (错误信息: {e})')
   finally:
         download_queue.task_done()

# 在线程中执行下载任务
def download_in_thread():
global download_queue
if not os.path.exists(download_path):
   ensure_download_path()

# 清空队列并添加新的下载任务
download_queue = Queue()
for url in urls:
   file_name = url.split("/")[-1]
   download_queue.put((url, file_name))
   update_status(f"添加下载任务: {file_name}")

# 创建并启动下载线程
threads = []
for _ in range(thread_count):# 创建线程处理所有任务
   thread = Thread(target=thread_download_files)
   thread.start()
   threads.append(thread)

# 等待所有下载线程完成
for thread in threads:
   thread.join()

# 所有任务完成后更新状态栏
root.after(0, lambda: update_status("所有下载任务完成！"))

# 开始下载
def start_download():
global urls, download_queue
if not urls:
   messagebox.showerror("错误", "没有可下载的URL。")
   return
Thread(target=download_in_thread).start()

# 创建GUI窗口
root.title("Topaz模型辅助下载器")

# 创建状态栏
status_frame = tk.Frame(root, bd=2, relief=tk.SUNKEN)
status_label = tk.Label(status_frame, text="就绪。", anchor="w")
status_label.pack(side="left", fill="x", padx=5, pady=5)
status_frame.pack(side="bottom", fill="x")

# 创建TXT路径相关控件
txt_path_frame = tk.Frame(root)
lbl_txt_path = tk.Label(txt_path_frame, text="TXT路径:")
lbl_txt_path.pack(side="left", padx=5, pady=5, anchor="w")
txt_txt_path = tk.Entry(txt_path_frame, width=50)
txt_txt_path.insert(0, txt_path)
txt_txt_path.pack(side="left", padx=5, pady=5, fill="x", expand=True)
btn_select_txt = tk.Button(txt_path_frame, text="浏览TXT", command=select_txt_file)
btn_select_txt.pack(side="left", padx=5, pady=5)

# 创建保存路径相关控件
download_path_frame = tk.Frame(root)
lbl_download_path = tk.Label(download_path_frame, text="保存路径:")
lbl_download_path.pack(side="left", padx=5, pady=5, anchor="w")
txt_download_path = tk.Entry(download_path_frame, width=50)
txt_download_path.insert(0, download_path)
txt_download_path.pack(side="left", padx=5, pady=5, fill="x", expand=True)
btn_select_download_path = tk.Button(download_path_frame, text="选择路径", command=select_download_path)
btn_select_download_path.pack(side="left", padx=5, pady=5)
btn_start_download = tk.Button(download_path_frame, text="开始下载", command=start_download)
btn_start_download.pack(side="left", padx=5, pady=5)

# 打包TXT路径相关控件
txt_path_frame.pack(side="top", fill="x", padx=5, pady=5)

# 打包保存路径相关控件
download_path_frame.pack(side="top", fill="x", padx=5, pady=5)

# 运行GUI
root.mainloop()

kakrate 发表于 2024-3-28 13:12

w759003376 发表于 2024-3-30 13:17

本帖最后由 w759003376 于 2024-3-30 14:05 编辑

这边调试了下，发现卡的地方就是def download_file(url, file_name): 这个方法里的update_status 方法，去掉或者换成print就不卡了
个人理解原因：
1. 因为update_status 方法里的函数status_label.config(text=text)并不会立即执行，而是会等调用它的线程结束才会执行
2.thread.join的原因，所以这个线程又再等待函数status_label.config(text=text)结束，所以互相等待，导致死锁

abpyu 发表于 2024-3-30 19:50

w759003376 发表于 2024-3-30 13:17
这边调试了下，发现卡的地方就是def download_file(url, file_name): 这个方法里的update_status 方法， ...

谢谢大佬回复，还在学习中{:1_893:}

页: [1]

吾爱破解 - 52pojie.cn's Archiver

Python 多线程(多任务)下载假死求助