【2021最新】AzureのBing Search APIで大量に画像をPythonでスクレイピングする方法

# -*- coding:utf-8 -*-
import requests
import matplotlib.pyplot as plt
from PIL import Image
from io import BytesIO
import math
import os
import time
import urllib
import hashlib
import argparse
import sys


SUBSCRIPTION_KEY = "INPUT APIKEY"
ENDPOINT = "https://api.bing.microsoft.com/v7.0/images/search"
path = "getImages"
headers = {"Ocp-Apim-Subscription-Key" : SUBSCRIPTION_KEY, 'Content-Type': 'multipart/form-data'}


def get_args():
# オブジェクト生成
parser = argparse.ArgumentParser()
if sys.stdin.isatty():
parser.add_argument("-q", "--query", type=str, required=True, nargs='*')
parser.add_argument("-c", "--count", type=int, default=3)
parser.add_argument("-n", "--reqnum", type=int, default=1)
args = parser.parse_args()
return(args)


def mkdir(path):
if not os.path.exists(path):
os.makedirs(path)


# 引数fをファイル名と拡張子（.は含まない）に分割する
def split_filename(f):
split_name = os.path.splitext(f)
file_name =split_name[0]
extension = split_name[-1].replace(".","")
return file_name,extension


def download_img(path,quary,id,url):
mkdir(path)
_,extension = split_filename(url)
if extension.lower() in ('jpg','jpeg','gif','png','bmp'):
encode_url = urllib.parse.unquote(url).encode('utf-8')
filename = '_'.join(quary)+"_"+id
full_path = os.path.join(path,filename + '.' + extension.lower())


r = requests.get(url, allow_redirects=True, timeout=10)
if r.status_code == requests.codes.ok:
with open(full_path,'wb') as f:
f.write(r.content)
print('【INFO】saved image...{}'.format(url))
else:
print("【ERROR】HttpError:{0} at{1}".format(r.status_code,image_url))


if __name__ == "__main__":
args = get_args()


# パラメータ
# license: パブリックドメインで画像を検索する
# imageType: photo指定で写真のみに絞る
query = args.query
count = args.count # 1リクエストあたりの最大取得件数 default:35 max:150
mkt = "ja-JP" # 取得元の国コード。指定すると応答が最適化される
num_per = args.reqnum # リクエスト回数（count * num_per=取得画像数）
offset = math.floor(count / num_per) # ループ回数。countの数でページング


for offset_num in range(offset):
params = {'q':query,'count':count,'offset':offset_num*offset,'mkt':mkt}
res = requests.get(ENDPOINT,headers=headers,params=params)
res.raise_for_status()
data = res.json()


for values in data['value'][:16]:
image_id = values['imageId']
image_url = values['contentUrl']
try:
download_img(path,query,image_id,image_url)
except Exception as e:
print("【INFO】failed to download image at {}".format(image_url))
print(e)
time.sleep(0.5)