HTTP编程不仅仅是发送简单的GET/POST请求,还涉及性能优化、安全实践、高级协议特性等。
1. 连接管理与性能优化
连接池管理
import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry # 创建自定义会话 session = requests.Session() # 配置重试策略 retry_strategy = Retry( total=3, backoff_factor=1, status_forcelist=[408, 429, 500, 502, 503, 504] ) # 配置适配器 adapter = HTTPAdapter( max_retries=retry_strategy, pool_connections=10, # 连接池大小 pool_maxsize=10 ) # 挂载适配器 session.mount("http://", adapter) session.mount("https://", adapter)
异步HTTP请求
import aiohttp import asyncio async def fetch(session, url): async with session.get(url) as response: return await response.text() async def main(): async with aiohttp.ClientSession() as session: html = await fetch(session, 'http://example.com') print(html) asyncio.run(main())
2. 安全实践
HTTPS证书验证
# 严格证书验证 response = requests.get('https://example.com', verify=True) # 自定义CA包 response = requests.get('https://example.com', verify='/path/to/cacert.pem') # 证书双向验证 response = requests.get( 'https://example.com', cert=('/path/client.cert', '/path/client.key'), verify='/path/to/cacert.pem' )
安全头部处理
# 设置安全头部 headers = { 'Content-Security-Policy': "default-src 'self'", 'X-Content-Type-Options': 'nosniff', 'X-Frame-Options': 'DENY', 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains; preload' }
3. 高级协议特性
HTTP/2支持
import httpx # 使用支持HTTP/2的客户端 with httpx.Client(http2=True) as client: response = client.get('https://http2.pro/api/v1') print(response.http_version) # 输出: "HTTP/2"
WebSocket通信
import websockets import asyncio async def websocket_client(): async with websockets.connect('ws://example.com/ws') as websocket: await websocket.send("Hello Server!") response = await websocket.recv() print(f"Received: {response}") asyncio.get_event_loop().run_until_complete(websocket_client())
4. 高级请求处理
流式请求与响应
# 流式上传大文件 def generate_large_file(): for i in range(1000): yield f"data chunk {i}\n".encode() requests.post('http://example.com/upload', data=generate_large_file()) # 流式下载 response = requests.get('http://example.com/large_file', stream=True) for chunk in response.iter_content(chunk_size=8192): process_chunk(chunk)
多部分表单高级用法
# 复杂多部分表单 files = { 'file': ('report.xls', open('report.xls', 'rb'), 'application/vnd.ms-excel'), 'metadata': ('metadata.json', json.dumps({'author': 'John'}), 'application/json') } requests.post('http://example.com/upload', files=files)
5. 监控与调试
请求追踪
# 启用调试日志 import logging import http.client http.client.HTTPConnection.debuglevel = 1 logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True requests.get('https://example.com')
性能分析
import time from pyinstrument import Profiler profiler = Profiler() profiler.start() # 执行HTTP请求 response = requests.get('https://example.com') profiler.stop() print(profiler.output_text(unicode=True, color=True))
6. 协议扩展
GRPC集成
import grpc import my_proto_pb2 import my_proto_pb2_grpc channel = grpc.insecure_channel('localhost:50051') stub = my_proto_pb2_grpc.MyServiceStub(channel) response = stub.MyMethod(my_proto_pb2.MyRequest(name='value'))
GraphQL请求
query = """ { user(id: "1") { name email } } """ response = requests.post( 'https://api.example.com/graphql', json={'query': query} )
7. 测试与Mock
请求Mock测试
from unittest.mock import patch def test_http_call(): with patch('requests.get') as mock_get: mock_get.return_value.status_code = 200 mock_get.return_value.json.return_value = {'key': 'value'} response = requests.get('http://example.com/api') assert response.status_code == 200 assert response.json()['key'] == 'value'
使用responses库
import responses @responses.activate def test_my_api(): responses.add( responses.GET, 'http://example.com/api', json={'error': 'not found'}, status=404 ) response = requests.get('http://example.com/api') assert response.status_code == 404 assert response.json()['error'] == 'not found'
实践总结
-
连接管理:使用会话和连接池提高性能
-
错误处理:实现健壮的重试机制
-
安全优先:强制HTTPS、验证证书、设置安全头部
-
协议支持:根据需求选择HTTP/1.1、HTTP/2或WebSocket
-
资源优化:对大文件使用流式处理
-
可观测性:添加日志和监控
-
测试覆盖:使用Mock进行HTTP调用测试