Python: генератор Sitemap Index — разбивка на 50 000 URL и gzip-архивы
Скрипт генерирует *sitemapindex.xml* и пачку *sitemap.xml.gz*, автоматически бьёт список URL на чанки по 50 000 и сжимает файлы.
# sitemap_gen.py
from pathlib import Path
from xml.etree.ElementTree import Element, SubElement, ElementTree
from datetime import datetime
import gzip
def write_sitemap(urls, path):
urlset = Element('urlset', {
'xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9'
})
now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
for u in urls:
url = SubElement(urlset, 'url')
SubElement(url, 'loc').text = u
SubElement(url, 'lastmod').text = now
SubElement(url, 'changefreq').text = 'weekly'
SubElement(url, 'priority').text = '0.6'
xml_bytes = ElementTree(urlset)
tmp = path.with_suffix('') # без .gz
xml_bytes.write(tmp, encoding='utf-8', xml_declaration=True)
with open(tmp, 'rb') as src, gzip.open(path, 'wb', compresslevel=6) as dst:
dst.write(src.read())
tmp.unlink()
def generate_sitemaps(all_urls, base_url, out_dir='public', chunk_size=50_000):
out = Path(out_dir); out.mkdir(parents=True, exist_ok=True)
sitemap_files = []
for i in range(0, len(all_urls), chunk_size):
chunk = all_urls[i:i+chunk_size]
gz_path = out / f'sitemap_{i//chunk_size + 1}.xml.gz'
write_sitemap(chunk, gz_path)
sitemap_files.append(gz_path.name)
# sitemap index
smindex = Element('sitemapindex', {
'xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9'
})
now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
for name in sitemap_files:
sm = SubElement(smindex, 'sitemap')
SubElement(sm, 'loc').text = f'{base_url.rstrip("/")}/{name}'
SubElement(sm, 'lastmod').text = now
ElementTree(smindex).write(out / 'sitemap_index.xml',
encoding='utf-8', xml_declaration=True)
# ─── Пример ───
if __name__ == '__main__':
urls = [f'https://example.com/snippet/{i}' for i in range(1, 123456)]
generate_sitemaps(urls, base_url='https://example.com', out_dir='public')