File size: 5,610 Bytes
c59fd93 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | #!/usr/bin/env python3
import urllib.request
import os
import sys
import subprocess
HTTPLIB_VERSION = "refs/tags/v0.53.1"
# used by examples/gguf-hash, these repos have no release tag, so we pin a commit
XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"
SHA1_COMMIT = "e1e2536fcf6a8f9703be8c85d58724b408552287"
SHA256_COMMIT = "5e637272c13f200872d55ff579f7e2ab6c3f252f"
ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1"
vendor = {
"https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp",
"https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp",
"https://raw.githubusercontent.com/nothings/stb/refs/heads/master/stb_image.h": "vendor/stb/stb_image.h",
# not using latest tag to avoid this issue: https://github.com/ggml-org/llama.cpp/pull/17179#discussion_r2515877926
# "https://github.com/mackron/miniaudio/raw/refs/tags/0.11.24/miniaudio.h": "vendor/miniaudio/miniaudio.h",
"https://github.com/mackron/miniaudio/raw/9634bedb5b5a2ca38c1ee7108a9358a4e233f14d/miniaudio.h": "vendor/miniaudio/miniaudio.h",
f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/httplib.h": "httplib.h",
f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/split.py": "split.py",
f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE": "vendor/cpp-httplib/LICENSE",
"https://raw.githubusercontent.com/sheredom/subprocess.h/0dccaa9aa176dd6d7ef8afeca3c18d6e80a32795/subprocess.h": "vendor/sheredom/subprocess.h",
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c": "vendor/hash/xxhash/xxhash.c",
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h": "vendor/hash/xxhash/xxhash.h",
f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE": "vendor/hash/xxhash/LICENSE",
# clibs/sha1 ships no license file, the source header says public domain
f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c",
f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h",
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c",
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h",
f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE": "vendor/hash/sha256/LICENSE",
f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h",
f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md": "vendor/hash/rotate-bits/LICENSE.md",
}
# local changes kept on top of the upstream sources
patches = {
"vendor/hash/xxhash/xxhash.h": [(
'#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n',
'/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n'
' before allowing the windows compiler to use the C11 form.\n'
' Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n'
'#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n'
' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
)],
# sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.
# we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.
"vendor/hash/sha1/sha1.h": [
(
'#if defined(__cplusplus)\n'
'extern "C" {\n'
'#endif\n',
'namespace vendor_hash {\n'
),
(
'#if defined(__cplusplus)\n'
'}\n'
'#endif\n',
'} // namespace vendor_hash\n'
),
],
"vendor/hash/sha1/sha1.c": [
(
'#include "sha1.h"\n',
'#include "sha1.h"\n'
'\n'
'namespace vendor_hash {\n'
),
(
' SHA1Final((unsigned char *)hash_out, &ctx);\n'
'}\n',
' SHA1Final((unsigned char *)hash_out, &ctx);\n'
'}\n'
'\n'
'} // namespace vendor_hash\n'
),
],
# silence a maybe-uninitialized warning
"vendor/hash/sha256/sha256.c": [(
" uint32_t W[16];\n",
" uint32_t W[16] = {0};\n"
)],
}
for url, filename in vendor.items():
print(f"downloading {url} to {filename}") # noqa: NP100
urllib.request.urlretrieve(url, filename)
for filename, replacements in patches.items():
print(f"patching {filename}") # noqa: NP100
with open(filename, "r", encoding="utf-8", newline="") as f:
content = f.read()
for old, new in replacements:
if content.count(old) != 1:
print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100
sys.exit(1)
content = content.replace(old, new)
with open(filename, "w", encoding="utf-8", newline="") as f:
f.write(content)
print("Splitting httplib.h...") # noqa: NP100
try:
subprocess.check_call([
sys.executable, "split.py",
"--extension", "cpp",
"--out", "vendor/cpp-httplib"
])
except Exception as e:
print(f"Error: {e}") # noqa: NP100
sys.exit(1)
finally:
os.remove("split.py")
os.remove("httplib.h")
|