Multiprocessing vs Multithreading

I need to download files fast. I want to swamp my network cables to download as much as it can consume. In the past I've used multiprocessing because it was easy to split jobs to several processors. But it is more efficient to use multithreading.

Multithreading uses one processor, but you can set up a queue so that it will continuously run as much as it can handle.

As an example, here is some code...

# Brian Blaylock
# March 24, 2017                           Yesterday Salt Lake had lots of rain

"""
Fast download of HRRR grib2 files with threading
"""

from queue import Queue 
from threading import Thread
from datetime import datetime, timedelta
import numpy as np
import urllib2
import re
import os

var='TMP:2 m'

def download(URL):
    # Code to download something based on a URL

    # ...

def worker():
    while True:
        item = q.get()
        print "number:", item
        download(item)
        q.task_done()

num_of_threads = 10

q = Queue()
for i in range(num_of_threads):
    t = Thread(target=worker)
    t.daemon = True
    t.start()

# List of URL's to download from

URL_list = ['...','...','...']

timer1 = datetime.now()
for item in URL_list:
    q.put(item)

q.join()       # block until all tasks are done

There is speed up when downloading, and the copper wires are saturated with as much as it can handle with about 10 threads or ten processors.
Here is a visual of what I learned:

Download Satellite Images from NASA Worldview, Add a watermark with the date

Partly taken form some old "watermark" code I found several years ago, this script allows you do download MODIS images from NASA's Worldview image viewer. I then compiled the images into an animated GIF.

# Brian Blaylock
# University of Utah

# Download True Color Images from NASA WorldView
# and add the time stamp to the image

## Images downloaded from NASA's Worldview
## https://earthdata.nasa.gov/labs/worldview/
# 1) zoom in to desired location
# 2) click "take a snapshot" icon
# 3) Draw the region you wish to take a photo
# 4) Modify the URL below, particularly the lat and lon
# 5) Edit the start and end dates

import urllib
from datetime import datetime, timedelta
import numpy as np

from PIL import Image, ImageDraw, ImageFont, ImageEnhance

FONT = 'Arial.ttf'
watermark_label = "THE DATE GOES HERE"


def add_watermark(in_file, text, out_file='watermark.jpg', angle=0, opacity=0.8):
  img = Image.open(in_file).convert('RGB')
  watermark = Image.new('RGBA', img.size, (0,0,0,0))
  size = 2
  n_font = ImageFont.truetype(FONT, size)
  n_width, n_height = n_font.getsize(text)
  text_size_scale = 1.5
  while n_width+n_height < watermark.size[0]/text_size_scale:
    size += 2
    n_font = ImageFont.truetype(FONT, size)
    n_width, n_height = n_font.getsize(text)
  draw = ImageDraw.Draw(watermark, 'RGBA')
  draw.text(((watermark.size[0] - n_width) / 10,
            (watermark.size[1] - n_height) / 100),
  text, font=n_font)
  watermark = watermark.rotate(angle,Image.BICUBIC)
  alpha = watermark.split()[3 ]
  alpha = ImageEnhance.Brightness(alpha).enhance(opacity)
  watermark.putalpha(alpha)
  Image.composite(watermark, img, watermark,).save(out_file, 'JPEG')

##--------------------------------------

outdir = 'satellite_img/'

start_date = datetime(2015,8,15)
end_date = datetime(2015,8,25)


# specify the dates you want to retrieve

date = start_date

while end_date >= date:
  for sat in np.array(["Terra","Aqua"]):
    year = str(date.year)
    dayofyear = str(date.timetuple().tm_yday)
    stringdate = datetime.strftime(date,"%Y-%m-%d")

    URL = "http://map2.vis.earthdata.nasa.gov/image-downloadTIME="+year+dayofyear+"&extent=-116.6972484336448,35.054396923451016,-106.9589671836448,43.896193798451016&epsg=4326&layers=MODIS_"+sat+"_CorrectedReflectance_TrueColor,Coastlines,Reference_Features,MODIS_Fires_All&opacities=1,1,1,1&worldfile=false&format=image/jpeg&width=4433&height=4024"

    #Since the Terra satellite passes over before Aqua
    # we need to save it before Aqua (alphebetical order puts it in wrong order)
    if sat == "Terra":
      sat_order="1Terra"
    if sat == "Aqua":
      sat_order="2Aqua"
    image_name = 'MODIS_TrueColor_'+stringdate+'_'+sat_order+'.jpg'
    urllib.urlretrieve(URL, outdir+image_name)
    add_watermark(outdir+image_name,'MODIS '+sat+' True Color: '+stringdate,outdir+image_name)
    print "Saved", image_name
  date = date+timedelta(days=1)