Percent-decode filenames and use index.html if there's no filename in the URL

  Merge https://salsa.debian.org/debian/wcurl/-/merge_requests/4

 closes #10
 closes #4

Author: samueloph

README.md

@@ -16,8 +16,8 @@ SPDX-License-Identifier: curl
 
 # Synopsis
 
-    wcurl [--curl-options <CURL_OPTIONS>]... [--dry-run] [--] <URL>...
-    wcurl [--curl-options=<CURL_OPTIONS>]... [--dry-run] [--] <URL>...
+    wcurl [--curl-options <CURL_OPTIONS>]... [--dry-run] [--no-decode-filename] [--] <URL>...
+    wcurl [--curl-options=<CURL_OPTIONS>]... [--dry-run] [--no-decode-filename] [--] <URL>...
     wcurl -V|--version
     wcurl -h|--help
 
@@ -35,14 +35,16 @@ should be using curl directly if your use case is not covered.
 
 
 * By default, **wcurl** will:
-  * Encode whitespaces in URLs;
+  * Percent-encode whitespaces in URLs;
   * Download multiple URLs in parallel if the installed curl's version is >= 7.66.0;
   * Follow redirects;
   * Automatically choose a filename as output;
   * Avoid overwriting files if the installed curl's version is >= 7.83.0 (`--no-clobber`);
   * Perform retries;
   * Set the downloaded file timestamp to the value provided by the server, if available;
   * Disable **curl**'s URL globbing parser so `{}` and `[]` characters in URLs are not treated specially.
+  * Percent-decode the resulting filename.
+  * Use "index.html" as default filename if there's none in the URL.
 
 # Options
 
@@ -51,6 +53,9 @@ should be using curl directly if your use case is not covered.
 
   Specify extra options to be passed when invoking curl. May be specified more than once.
 
+* `--no-decode-filename`
+  Don't percent-decode the output filename, even if the percent-encoding in the URL was done by wcurl, e.g.: The URL contained whitespaces.
+
 * `--dry-run`
 
   Don't actually execute curl, just print what would be invoked.
@@ -66,7 +71,7 @@ should be using curl directly if your use case is not covered.
 # Url
 
 Anything which is not a parameter will be considered an URL.
-**wcurl** will encode whitespaces and pass that to curl, which will perform the
+**wcurl** will percent-encode whitespaces and pass that to curl, which will perform the
 parsing of the URL.
 
 # Examples

tests/tests.sh

@@ -112,6 +112,83 @@ testUrlStartingWithDash()
     assertEquals "${ret}" "Unknown option: '-example.com'."
 }
 
+testUrlDefaultName()
+{
+    url='example%20with%20spaces.com'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' chooses the correct default filename when there's no path in the URL" "${ret}" 'index.html'
+}
+
+testUrlDefaultNameTrailingSlash()
+{
+    url='example%20with%20spaces.com/'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' chooses the correct default filename when there's no path in the URL and the URl ends with a slash" "${ret}" 'index.html'
+}
+
+testUrlDecodingWhitespaces()
+{
+    url='example.com/filename%20with%20spaces'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded whitespaces in URLs" "${ret}" 'filename with spaces'
+}
+
+testUrlDecodingWhitespacesTwoFiles()
+{
+    url='example.com/filename%20with%20spaces'
+    url_2='example.com/filename2%20with%20spaces'
+    ret=$(${WCURL_CMD} ${url} ${url_2} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded whitespaces in URLs" "${ret}" 'filename with spaces'
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded whitespaces in URLs" "${ret}" 'filename2 with spaces'
+}
+
+testUrlDecodingDisabled()
+{
+    url='example.com/filename%20with%20spaces'
+    ret=$(${WCURL_CMD} --no-decode-filename ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded whitespaces in URLs" "${ret}" 'filename%20with%20spaces'
+}
+
+testUrlDecodingWhitespacesQueryString()
+{
+    url='example.com/filename%20with%20spaces?query=string'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded whitespaces in URLs with query strings" "${ret}" 'filename with spaces'
+}
+
+testUrlDecodingWhitespacesTrailingSlash()
+{
+    url='example.com/filename%20with%20spaces/'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully uses the default filename when the URL ends with a slash" "${ret}" 'index.html'
+}
+
+# Test decoding a bunch of different languages (that don't use the latin
+# alphabet), we could split each language on its own test, but for now it
+# doesn't make a difference.
+testUrlDecodingNonLatinLanguages()
+{
+    # Arabic
+    url='example.com/%D8%AA%D8%B1%D9%85%D9%8A%D8%B2_%D8%A7%D9%84%D9%86%D8%B3%D8%A8%D8%A9_%D8%A7%D9%84%D9%85%D8%A6%D9%88%D9%8A%D8%A9'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded Arabic in URLs" "${ret}" 'ترميز_النسبة_المئوية'
+
+    # Persian
+    url='example.com/%DA%A9%D8%AF%D8%A8%D9%86%D8%AF%DB%8C_%D8%AF%D8%B1%D8%B5%D8%AF%DB%8C'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded Persian in URLs" "${ret}" 'کدبندی_درصدی'
+
+    # Japanese
+    url='example.com/%E3%83%91%E3%83%BC%E3%82%BB%E3%83%B3%E3%83%88%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%87%E3%82%A3%E3%83%B3%E3%82%B0'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded Japanese in URLs" "${ret}" 'パーセントエンコーディング'
+
+    # Korean
+    url='example.com/%ED%8D%BC%EC%84%BC%ED%8A%B8_%EC%9D%B8%EC%BD%94%EB%94%A9'
+    ret=$(${WCURL_CMD} ${url} 2>&1)
+    assertContains "Verify whether 'wcurl' successfully decodes percent-encoded Korean in URLs" "${ret}" '퍼센트_인코딩'
+}
+
 ## Ideas for tests:
 ##
 ## - URL with whitespace

wcurl

@@ -48,8 +48,8 @@ usage()
     cat << _EOF_
 ${PROGRAM_NAME} -- a simple wrapper around curl to easily download files.
 
-Usage: ${PROGRAM_NAME} [--curl-options <CURL_OPTIONS>] [--dry-run] [--] <URL>...
-       ${PROGRAM_NAME} [--curl-options=<CURL_OPTIONS>] [--dry-run] [--] <URL>...
+Usage: ${PROGRAM_NAME} [--curl-options <CURL_OPTIONS>] [--dry-run] [--no-decode-filename] [--] <URL>...
+       ${PROGRAM_NAME} [--curl-options=<CURL_OPTIONS>] [--dry-run] [--no-decode-filename] [--] <URL>...
        ${PROGRAM_NAME} -h|--help
        ${PROGRAM_NAME} -V|--version
 
@@ -59,14 +59,19 @@ Options:
                                  passed when invoking curl. May be
                                  specified more than once.
 
+  --no-decode-filename: Don't percent-decode the output filename,
+                        even if the percent-encoding in the URL was
+                        done by wcurl, e.g.: The URL contained
+                        whitespaces.
+
   --dry-run: Don't actually execute curl, just print what would be
              invoked.
 
   -V,--version: Print version information.
 
   -h,--help: Print this usage message.
 
-  <URL>: The URL to be downloaded.  May be specified more than once.
+  <URL>: The URL to be downloaded. May be specified more than once.
 _EOF_
 }
 
@@ -93,7 +98,6 @@ readonly PER_URL_PARAMETERS="\
     --globoff \
     --location \
     --proto-default https \
-    --remote-name-all \
     --remote-time \
     --retry 10 \
     --retry-max-time 10 "
@@ -111,6 +115,56 @@ sanitize()
     readonly CURL_OPTIONS URLS DRY_RUN
 }
 
+# Indicate via exit code whether the string given in the first parameter
+# consists solely of characters from the string given in the second parameter.
+# In other words, it returns 0 if the first parameter only contains characters
+# from the second parameter, e.g.: Are $1 characters a subset of $2 characters?
+is_subset_of()
+{
+    case "${1}" in
+        *[!${2}]*|'') return 1;;
+    esac
+}
+
+# Print the given string percent-decoded.
+percent_decode()
+{
+    # Encodings of control characters (00-1F) are passed through without decoding.
+    # Iterate on the input character-by-character, decoding it.
+    printf "%s\n" "${1}" | fold -w1 | while IFS= read -r decode_out; do
+        # If character is a "%", read the next character as decode_hex1.
+        if [ "${decode_out}" = % ] && IFS= read -r decode_hex1; then
+            decode_out="${decode_out}${decode_hex1}"
+            # If there's one more character, read it as decode_hex2.
+            if IFS= read -r decode_hex2; then
+                decode_out="${decode_out}${decode_hex2}"
+                # Skip decoding if this is a control character (00-1F).
+                # Skip decoding if DECODE_FILENAME is not "true".
+                if is_subset_of "${decode_hex1}" "23456789abcdefABCDEF" && \
+                    is_subset_of "${decode_hex2}" "0123456789abcdefABCDEF" && \
+                    [ "${DECODE_FILENAME}" = "true" ]; then
+                    # Use printf to decode it into octal and then decode it to the final format.
+                    decode_out="$(printf "%b" "\\$(printf %o "0x${decode_hex1}${decode_hex2}")")"
+                fi
+            fi
+        fi
+        printf %s "${decode_out}"
+    done
+}
+
+# Print the percent-decoded filename portion of the given URL.
+get_url_filename()
+{
+    # Remove protocol and query string if present.
+    hostname_and_path="$(printf %s "${1}" | sed -e 's,^[^/]*//,,' -e 's,?.*$,,')"
+    # If what remains contains a slash, there's a path; return it percent-decoded.
+    case "${hostname_and_path}" in
+        # sed to remove everything preceeding the last '/', e.g.: "example/something" becomes "something"
+        */*) percent_decode "$(printf %s "${hostname_and_path}" | sed -e 's,^.*/,,')";;
+    esac
+    # No slash means there was just a hostname and no path; return empty string.
+}
+
 # Execute curl with the list of URLs provided by the user.
 exec_curl()
 {
@@ -159,8 +213,10 @@ exec_curl()
 
     NEXT_PARAMETER=""
     for url in ${URLS}; do
+        filename="$(get_url_filename "${url}")"
+        [ -z "${filename}" ] && filename=index.html
         # shellcheck disable=SC2086
-        set -- "$@" ${NEXT_PARAMETER} ${PER_URL_PARAMETERS} ${CURL_HAS_NO_CLOBBER} ${CURL_OPTIONS} "${url}"
+        set -- "$@" ${NEXT_PARAMETER} ${PER_URL_PARAMETERS} ${CURL_HAS_NO_CLOBBER} ${CURL_OPTIONS} --output "${filename}" "${url}"
         NEXT_PARAMETER="--next"
     done
 
@@ -171,6 +227,9 @@ exec_curl()
     fi
 }
 
+# Default to decoding the output filename
+DECODE_FILENAME="true"
+
 # Use "${1-}" in order to avoid errors because of 'set -u'.
 while [ -n "${1-}" ]; do
     case "${1}" in
@@ -188,6 +247,10 @@ while [ -n "${1-}" ]; do
             DRY_RUN="true"
             ;;
 
+        --no-decode-filename)
+            DECODE_FILENAME="false"
+            ;;
+
         -h|--help)
             usage
             exit 0

wcurl.1

@@ -27,8 +27,8 @@
 - a simple wrapper around curl to easily download files.
 .SH SYNOPSIS
 .nf
-\fBwcurl [\-\-curl\-options \fI<CURL_OPTIONS>\fP]... [\-\-dry\-run] [\-\-] \fI<URL>\fP...\fR
-\fBwcurl [\-\-curl\-options=\fI<CURL_OPTIONS>\fP]... [\-\-dry\-run] [\-\-] \fI<URL>\fP...\fR
+\fBwcurl [\-\-curl\-options \fI<CURL_OPTIONS>\fP]... [\-\-dry\-run] [\-\-no\-decode\-filename] [\-\-] \fI<URL>\fP...\fR
+\fBwcurl [\-\-curl\-options=\fI<CURL_OPTIONS>\fP]... [\-\-dry\-run] [\-\-no\-decode\-filename] [\-\-] \fI<URL>\fP...\fR
 \fBwcurl \-V|\-\-version\fR
 \fBwcurl \-h|\-\-help\fR
 .fi
@@ -46,7 +46,7 @@ should be using curl directly if your use case is not covered.
 .TP
 By default, \fBwcurl\fR will:
 .br
-\[bu]  Encode whitespaces in URLs;
+\[bu]  Percent-encode whitespaces in URLs;
 .br
 \[bu]  Download multiple URLs in parallel if the installed curl's version is >= 7.66.0;
 .br
@@ -63,6 +63,10 @@ By default, \fBwcurl\fR will:
 \[bu]  Default to the protocol used as https if the URL doesn't contain any;
 .br
 \[bu]  Disable \fBcurl\fR's URL globbing parser so \fB{}\fR and \fB[]\fR characters in URLs are not treated specially.
+.br
+\[bu]  Percent-decode the resulting filename.
+.br
+\[bu]  Use "index.html" as default filename if there's none in the URL.
 .SH OPTIONS
 .TP
 \fB\-\-curl\-options, \-\-curl\-options=\fI<CURL_OPTIONS>\fR...\fR
@@ -81,7 +85,7 @@ Any option supported by curl can be set here.
 This is not used by \fBwcurl\fR; it's instead forwarded to the curl invocation.
 .SH URL
 Anything which is not a parameter will be considered an URL.
-\fBwcurl\fR will encode whitespaces and pass that to curl, which will perform the
+\fBwcurl\fR will percent-encode whitespaces and pass that to curl, which will perform the
 parsing of the URL.
 .SH EXAMPLES
 Download a single file: