1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
|
#
# Download and optionally unpack a file
#
# ocv_download(FILENAME p HASH h URL u1 [u2 ...] DESTINATION_DIR d [ID id] [STATUS s] [UNPACK] [RELATIVE_URL])
# FILENAME - filename
# HASH - MD5 hash
# URL - full download url (first nonempty value will be chosen)
# DESTINATION_DIR - file will be copied to this directory
# ID - identifier for project/group of downloaded files
# STATUS - passed variable will be updated in parent scope,
# function will not fail the build in case of download problem if this option is provided,
# but will fail in case when other operations (copy, remove, etc.) failed
# UNPACK - downloaded file will be unpacked to DESTINATION_DIR
# RELATIVE_URL - if set, then URL is treated as a base, and FILENAME will be appended to it
# Note: uses OPENCV_DOWNLOAD_PATH folder as cache, default is <opencv>/.cache
set(HELP_OPENCV_DOWNLOAD_PATH "Cache directory for downloaded files")
if(DEFINED ENV{OPENCV_DOWNLOAD_PATH})
set(OPENCV_DOWNLOAD_PATH "$ENV{OPENCV_DOWNLOAD_PATH}" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}")
endif()
set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}")
set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt")
set(OPENCV_DOWNLOAD_WITH_CURL "${OpenCV_BINARY_DIR}/download_with_curl.sh")
set(OPENCV_DOWNLOAD_WITH_WGET "${OpenCV_BINARY_DIR}/download_with_wget.sh")
set(OPENCV_DOWNLOAD_TRIES_LIST 1 CACHE STRING "List of download tries") # a list
set(OPENCV_DOWNLOAD_PARAMS INACTIVITY_TIMEOUT 60 TIMEOUT 600 CACHE STRING "Download parameters to be passed to file(DOWNLOAD ...)")
mark_as_advanced(OPENCV_DOWNLOAD_TRIES_LIST OPENCV_DOWNLOAD_PARAMS)
# Init download cache directory and log file and helper scripts
if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}")
file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH})
endif()
if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}/.gitignore")
file(WRITE "${OPENCV_DOWNLOAD_PATH}/.gitignore" "*\n")
endif()
file(WRITE "${OPENCV_DOWNLOAD_LOG}" "#use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n")
file(REMOVE "${OPENCV_DOWNLOAD_WITH_CURL}")
file(REMOVE "${OPENCV_DOWNLOAD_WITH_WGET}")
ocv_check_environment_variables(OPENCV_DOWNLOAD_MIRROR_ID)
function(ocv_init_download_mirror)
if(NOT DEFINED OPENCV_DOWNLOAD_MIRROR_ID)
# Run `git remote get-url origin` to get remote source
execute_process(
COMMAND
git remote get-url origin
WORKING_DIRECTORY
${CMAKE_SOURCE_DIR}
RESULT_VARIABLE
RESULT_STATUS
OUTPUT_VARIABLE
OCV_GIT_ORIGIN_URL_OUTPUT
ERROR_QUIET
)
# if non-git, OCV_GIT_ORIGIN_URL_OUTPUT is empty
if(NOT OCV_GIT_ORIGIN_URL_OUTPUT)
message(STATUS "ocv_init_download: OpenCV source tree is not fetched as git repository. 3rdparty resources will be downloaded from github.com by default.")
return()
else()
# Check if git origin is github.com
string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "github.com" _found_github)
if(NOT ${_found_github} EQUAL -1)
set(OPENCV_DOWNLOAD_MIRROR_ID "github" CACHE STRING "")
endif()
# Check if git origin is gitcode.net
string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "gitcode.net" _found_gitcode)
if(NOT ${_found_gitcode} EQUAL -1)
set(OPENCV_DOWNLOAD_MIRROR_ID "gitcode" CACHE STRING "")
endif()
endif()
endif()
if(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "gitcode" OR OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "custom")
message(STATUS "ocv_init_download: Using ${OPENCV_DOWNLOAD_MIRROR_ID}-hosted mirror to download 3rdparty components.")
ocv_cmake_hook_append(OPENCV_DOWNLOAD_PRE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/mirrors/${OPENCV_DOWNLOAD_MIRROR_ID}.cmake")
elseif(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "github")
return()
else()
message(STATUS "ocv_init_download: Unable to recognize git server of OpenCV source code. Using github.com to download 3rdparty components.")
endif()
endfunction()
function(ocv_download)
cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN})
function(ocv_download_log)
file(APPEND "${OPENCV_DOWNLOAD_LOG}" "${ARGN}\n")
endfunction()
ocv_assert(DL_FILENAME)
ocv_assert(DL_HASH)
ocv_assert(DL_URL)
ocv_assert(DL_DESTINATION_DIR)
if((NOT " ${DL_UNPARSED_ARGUMENTS}" STREQUAL " ")
OR DL_FILENAME STREQUAL ""
OR DL_HASH STREQUAL ""
OR DL_URL STREQUAL ""
OR DL_DESTINATION_DIR STREQUAL ""
)
set(msg_level FATAL_ERROR)
if(DEFINED DL_STATUS)
set(${DL_STATUS} FALSE PARENT_SCOPE)
set(msg_level WARNING)
endif()
message(${msg_level} "ERROR: ocv_download() unsupported arguments: ${ARGV}")
return()
endif()
if(DEFINED DL_STATUS)
set(${DL_STATUS} TRUE PARENT_SCOPE)
endif()
ocv_cmake_hook(OPENCV_DOWNLOAD_PRE)
# Check CMake cache for already processed tasks
string(FIND "${DL_DESTINATION_DIR}" "${CMAKE_BINARY_DIR}" DL_BINARY_PATH_POS)
if(DL_BINARY_PATH_POS EQUAL 0)
set(__file_id "${DL_DESTINATION_DIR}/${DL_FILENAME}")
file(RELATIVE_PATH __file_id "${CMAKE_BINARY_DIR}" "${__file_id}")
string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __file_id "${__file_id}")
if(DL_ID)
string(TOUPPER ${DL_ID} __id)
string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __id "${__id}")
set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_${__id}_HASH_${__file_id}")
else()
set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_HASH_${__file_id}")
endif()
if(" ${${OCV_DOWNLOAD_HASH_NAME}}" STREQUAL " ${DL_HASH}")
ocv_download_log("#match_hash_in_cmake_cache \"${OCV_DOWNLOAD_HASH_NAME}\"")
return()
endif()
unset("${OCV_DOWNLOAD_HASH_NAME}" CACHE)
else()
set(OCV_DOWNLOAD_HASH_NAME "")
#message(WARNING "Download destination is not in CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}: ${DL_DESTINATION_DIR}")
endif()
# Select first non-empty url
foreach(url ${DL_URL})
if(url)
set(DL_URL "${url}")
break()
endif()
endforeach()
# Append filename to url if needed
if(DL_RELATIVE_URL)
set(DL_URL "${DL_URL}${DL_FILENAME}")
endif()
set(mode "copy")
if(DL_UNPACK)
set(mode "unpack")
endif()
# Log all calls to file
ocv_download_log("#do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"")
# ... and to console
set(__msg_prefix "")
if(DL_ID)
set(__msg_prefix "${DL_ID}: ")
endif()
message(STATUS "${__msg_prefix}Downloading ${DL_FILENAME} from ${DL_URL}")
# Copy mode: check if copy destination exists and is correct
if(NOT DL_UNPACK)
set(COPY_DESTINATION "${DL_DESTINATION_DIR}/${DL_FILENAME}")
if(EXISTS "${COPY_DESTINATION}")
ocv_download_log("#check_md5 \"${COPY_DESTINATION}\"")
file(MD5 "${COPY_DESTINATION}" target_md5)
if(target_md5 STREQUAL DL_HASH)
ocv_download_log("#match_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"")
if(OCV_DOWNLOAD_HASH_NAME)
set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "")
endif()
return()
endif()
ocv_download_log("#mismatch_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"")
else()
ocv_download_log("#missing \"${COPY_DESTINATION}\"")
endif()
endif()
# Check cache first
if(DL_ID)
string(TOLOWER "${DL_ID}" __id)
string(REGEX REPLACE "[^a-zA-Z0-9_/ ]" "_" __id "${__id}")
set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${__id}/${DL_HASH}-${DL_FILENAME}")
else()
set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${DL_HASH}-${DL_FILENAME}")
endif()
if(EXISTS "${CACHE_CANDIDATE}")
ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"")
file(MD5 "${CACHE_CANDIDATE}" target_md5)
if(NOT target_md5 STREQUAL DL_HASH)
ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"")
ocv_download_log("#delete \"${CACHE_CANDIDATE}\"")
file(REMOVE ${CACHE_CANDIDATE})
endif()
endif()
# Download
if(NOT EXISTS "${CACHE_CANDIDATE}")
ocv_download_log("#cmake_download \"${CACHE_CANDIDATE}\" \"${DL_URL}\"")
foreach(try ${OPENCV_DOWNLOAD_TRIES_LIST})
ocv_download_log("#try ${try}")
file(DOWNLOAD "${DL_URL}" "${CACHE_CANDIDATE}"
STATUS status
LOG __log
${OPENCV_DOWNLOAD_PARAMS})
if(status EQUAL 0)
break()
endif()
message(STATUS "Try ${try} failed")
endforeach()
if(NOT OPENCV_SKIP_FILE_DOWNLOAD_DUMP) # workaround problem with old CMake versions: "Invalid escape sequence"
string(LENGTH "${__log}" __log_length)
if(__log_length LESS 65536)
string(REPLACE "\n" "\n# " __log "${__log}")
ocv_download_log("# ${__log}\n")
endif()
endif()
if(NOT status EQUAL 0)
set(msg_level FATAL_ERROR)
if(DEFINED DL_STATUS)
set(${DL_STATUS} FALSE PARENT_SCOPE)
set(msg_level WARNING)
endif()
if(status MATCHES "Couldn't resolve host name")
message(STATUS "
=======================================================================
Couldn't download files from the Internet.
Please check the Internet access on this host.
=======================================================================
")
elseif(status MATCHES "Couldn't connect to server")
message(STATUS "
=======================================================================
Couldn't connect to server from the Internet.
Perhaps direct connections are not allowed in the current network.
To use proxy please check/specify these environment variables:
- http_proxy/https_proxy
- and/or HTTP_PROXY/HTTPS_PROXY
=======================================================================
")
endif()
message(${msg_level} "${__msg_prefix}Download failed: ${status}
For details please refer to the download log file:
${OPENCV_DOWNLOAD_LOG}
")
# write helper scripts for failed downloads
file(APPEND "${OPENCV_DOWNLOAD_WITH_CURL}" "curl --create-dirs --output \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n")
file(APPEND "${OPENCV_DOWNLOAD_WITH_WGET}" "mkdir -p $(dirname ${CACHE_CANDIDATE}) && wget -O \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n")
return()
endif()
# Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step on wrong hash
ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"")
file(MD5 "${CACHE_CANDIDATE}" target_md5)
if(NOT target_md5 STREQUAL DL_HASH)
ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"")
set(msg_level FATAL_ERROR)
if(DEFINED DL_STATUS)
set(${DL_STATUS} FALSE PARENT_SCOPE)
set(msg_level WARNING)
endif()
message(${msg_level} "${__msg_prefix}Hash mismatch: ${target_md5}")
return()
endif()
endif()
# Unpack or copy
if(DL_UNPACK)
if(EXISTS "${DL_DESTINATION_DIR}")
ocv_download_log("#remove_unpack \"${DL_DESTINATION_DIR}\"")
file(REMOVE_RECURSE "${DL_DESTINATION_DIR}")
endif()
ocv_download_log("#mkdir \"${DL_DESTINATION_DIR}\"")
file(MAKE_DIRECTORY "${DL_DESTINATION_DIR}")
ocv_download_log("#unpack \"${DL_DESTINATION_DIR}\" \"${CACHE_CANDIDATE}\"")
execute_process(COMMAND "${CMAKE_COMMAND}" -E tar xzf "${CACHE_CANDIDATE}"
WORKING_DIRECTORY "${DL_DESTINATION_DIR}"
RESULT_VARIABLE res)
if(NOT res EQUAL 0)
message(FATAL_ERROR "${__msg_prefix}Unpack failed: ${res}")
endif()
else()
ocv_download_log("#copy \"${COPY_DESTINATION}\" \"${CACHE_CANDIDATE}\"")
execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CACHE_CANDIDATE}" "${COPY_DESTINATION}"
RESULT_VARIABLE res)
if(NOT res EQUAL 0)
message(FATAL_ERROR "${__msg_prefix}Copy failed: ${res}")
endif()
endif()
if(OCV_DOWNLOAD_HASH_NAME)
set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "")
endif()
endfunction()
# ----------------------------------------------------------------------------
# Initialize download in case mirror is used
# ----------------------------------------------------------------------------
ocv_init_download_mirror()
|