Skip to content

Commit 62ef030

Browse files
BillyONealBillyONeal
authored
Avoid curl command line length limits (#1390)
1 parent 87adc3f commit 62ef030

File tree

8 files changed

+245
-170
lines changed

8 files changed

+245
-170
lines changed

‎include/vcpkg/base/downloads.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ namespace vcpkg
3232

3333
View<std::string> azure_blob_headers();
3434

35-
std::vector<int> download_files(const Filesystem& fs,
36-
View<std::pair<std::string, Path>> url_pairs,
37-
View<std::string> headers);
35+
std::vector<int> download_files(View<std::pair<std::string, Path>> url_pairs,
36+
View<std::string> headers,
37+
View<std::string> secrets);
3838

3939
bool send_snapshot_to_api(const std::string& github_token,
4040
const std::string& github_repository,

‎include/vcpkg/base/message-data.inc.h‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,13 @@ DECLARE_MESSAGE(CommandFailed,
837837
"",
838838
"command:\n"
839839
"{command_line}\n"
840-
"failed with the following results:")
840+
"failed with the following output:")
841+
DECLARE_MESSAGE(CommandFailedCode,
842+
(msg::command_line, msg::exit_code),
843+
"",
844+
"command:\n"
845+
"{command_line}\n"
846+
"failed with exit code {exit_code} and the following output:")
841847
DECLARE_MESSAGE(CommunityTriplets, (), "", "Community Triplets:")
842848
DECLARE_MESSAGE(CompilerPath, (msg::path), "", "Compiler found: {path}")
843849
DECLARE_MESSAGE(CompressFolderFailed, (msg::path), "", "Failed to compress folder \"{path}\":")
@@ -902,10 +908,6 @@ DECLARE_MESSAGE(Creating7ZipArchive, (), "", "Creating 7zip archive...")
902908
DECLARE_MESSAGE(CreatingNugetPackage, (), "", "Creating NuGet package...")
903909
DECLARE_MESSAGE(CreatingZipArchive, (), "", "Creating zip archive...")
904910
DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.")
905-
DECLARE_MESSAGE(CurlFailedToExecute,
906-
(msg::exit_code),
907-
"curl is the name of a program, see curl.se",
908-
"curl failed to execute with exit code {exit_code}.")
909911
DECLARE_MESSAGE(CurlFailedToPut,
910912
(msg::exit_code, msg::url),
911913
"curl is the name of a program, see curl.se",
@@ -914,22 +916,15 @@ DECLARE_MESSAGE(CurlFailedToPutHttp,
914916
(msg::exit_code, msg::url, msg::value),
915917
"curl is the name of a program, see curl.se. {value} is an HTTP status code",
916918
"curl failed to put file to {url} with exit code {exit_code} and http code {value}.")
917-
DECLARE_MESSAGE(CurlReportedUnexpectedResults,
918-
(msg::command_line, msg::actual),
919-
"{command_line} is the command line to call curl.exe, {actual} is the console output "
920-
"of curl.exe locale-invariant download results.",
921-
"curl has reported unexpected results to vcpkg and vcpkg cannot continue.\n"
922-
"Please review the following text for sensitive information and open an issue on the "
923-
"Microsoft/vcpkg GitHub to help fix this problem!\n"
924-
"cmd: {command_line}\n"
925-
"=== curl output ===\n"
926-
"{actual}\n"
927-
"=== end curl output ===")
928-
DECLARE_MESSAGE(CurlReturnedUnexpectedResponseCodes,
929-
(msg::actual, msg::expected),
930-
"{actual} and {expected} are integers, curl is the name of a program, see curl.se",
931-
"curl returned a different number of response codes than were expected for the request ({actual} "
932-
"vs expected {expected}).")
919+
DECLARE_MESSAGE(CurlResponseTruncatedRetrying,
920+
(msg::value),
921+
"{value} is the number of milliseconds for which we are waiting this time",
922+
"curl returned a partial response; waiting {value} milliseconds and trying again")
923+
DECLARE_MESSAGE(CurlTimeout,
924+
(msg::command_line),
925+
"",
926+
"curl was unable to perform all requested HTTP operations, even after timeout and retries. The last "
927+
"command line was: {command_line}")
933928
DECLARE_MESSAGE(CurrentCommitBaseline,
934929
(msg::commit_sha),
935930
"",
@@ -2608,7 +2603,6 @@ DECLARE_MESSAGE(UnexpectedEOFMidArray, (), "", "Unexpected EOF in middle of arra
26082603
DECLARE_MESSAGE(UnexpectedEOFMidKeyword, (), "", "Unexpected EOF in middle of keyword")
26092604
DECLARE_MESSAGE(UnexpectedEOFMidString, (), "", "Unexpected EOF in middle of string")
26102605
DECLARE_MESSAGE(UnexpectedEOFMidUnicodeEscape, (), "", "Unexpected end of file in middle of unicode escape")
2611-
DECLARE_MESSAGE(UnexpectedErrorDuringBulkDownload, (), "", "an unexpected error occurred during bulk download.")
26122606
DECLARE_MESSAGE(UnexpectedEscapeSequence, (), "", "Unexpected escape sequence continuation")
26132607
DECLARE_MESSAGE(UnexpectedField, (msg::json_field), "", "unexpected field '{json_field}'")
26142608
DECLARE_MESSAGE(UnexpectedFieldSuggest,

‎include/vcpkg/base/system.process.h‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,26 +35,8 @@ namespace vcpkg
3535
explicit Command(StringView s) { string_arg(s); }
3636

3737
Command& string_arg(StringView s) &;
38-
Command& raw_arg(StringView s) &
39-
{
40-
if (!buf.empty())
41-
{
42-
buf.push_back(' ');
43-
}
44-
45-
buf.append(s.data(), s.size());
46-
return *this;
47-
}
48-
49-
Command& forwarded_args(View<std::string> args) &
50-
{
51-
for (auto&& arg : args)
52-
{
53-
string_arg(arg);
54-
}
55-
56-
return *this;
57-
}
38+
Command& raw_arg(StringView s) &;
39+
Command& forwarded_args(View<std::string> args) &;
5840

5941
Command&& string_arg(StringView s) && { return std::move(string_arg(s)); };
6042
Command&& raw_arg(StringView s) && { return std::move(raw_arg(s)); }
@@ -67,6 +49,13 @@ namespace vcpkg
6749
void clear() { buf.clear(); }
6850
bool empty() const { return buf.empty(); }
6951

52+
// maximum UNICODE_STRING, with enough space for one MAX_PATH prepended
53+
static constexpr size_t maximum_allowed = 32768 - 260 - 1;
54+
55+
// if `other` can be appended to this command without exceeding `maximum_allowed`, appends `other` and returns
56+
// true; otherwise, returns false
57+
bool try_append(const Command& other);
58+
7059
private:
7160
std::string buf;
7261
};

‎locales/messages.json‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -493,8 +493,10 @@
493493
"CmdZExtractOptStrip": "The number of leading directories to strip from all paths",
494494
"CommandEnvExample2": "vcpkg env \"ninja -C <path>\" --triplet x64-windows",
495495
"_CommandEnvExample2.comment": "This is a command line, only the <path> part should be localized",
496-
"CommandFailed": "command:\n{command_line}\nfailed with the following results:",
496+
"CommandFailed": "command:\n{command_line}\nfailed with the following output:",
497497
"_CommandFailed.comment": "An example of {command_line} is vcpkg install zlib.",
498+
"CommandFailedCode": "command:\n{command_line}\nfailed with exit code {exit_code} and the following output:",
499+
"_CommandFailedCode.comment": "An example of {command_line} is vcpkg install zlib. An example of {exit_code} is 127.",
498500
"CommunityTriplets": "Community Triplets:",
499501
"CompilerPath": "Compiler found: {path}",
500502
"_CompilerPath.comment": "An example of {path} is /foo/bar.",
@@ -538,16 +540,14 @@
538540
"CreatingZipArchive": "Creating zip archive...",
539541
"CreationFailed": "Creating {path} failed.",
540542
"_CreationFailed.comment": "An example of {path} is /foo/bar.",
541-
"CurlFailedToExecute": "curl failed to execute with exit code {exit_code}.",
542-
"_CurlFailedToExecute.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127.",
543543
"CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.",
544544
"_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
545545
"CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.",
546546
"_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
547-
"CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===",
548-
"_CurlReportedUnexpectedResults.comment": "{command_line} is the command line to call curl.exe, {actual} is the console output of curl.exe locale-invariant download results. An example of {command_line} is vcpkg install zlib.",
549-
"CurlReturnedUnexpectedResponseCodes": "curl returned a different number of response codes than were expected for the request ({actual} vs expected {expected}).",
550-
"_CurlReturnedUnexpectedResponseCodes.comment": "{actual} and {expected} are integers, curl is the name of a program, see curl.se",
547+
"CurlResponseTruncatedRetrying": "curl returned a partial response; waiting {value} milliseconds and trying again",
548+
"_CurlResponseTruncatedRetrying.comment": "{value} is the number of milliseconds for which we are waiting this time",
549+
"CurlTimeout": "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last command line was: {command_line}",
550+
"_CurlTimeout.comment": "An example of {command_line} is vcpkg install zlib.",
551551
"CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"",
552552
"_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
553553
"CycleDetectedDuring": "cycle detected during {spec}:",
@@ -1466,7 +1466,6 @@
14661466
"UnexpectedEOFMidKeyword": "Unexpected EOF in middle of keyword",
14671467
"UnexpectedEOFMidString": "Unexpected EOF in middle of string",
14681468
"UnexpectedEOFMidUnicodeEscape": "Unexpected end of file in middle of unicode escape",
1469-
"UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.",
14701469
"UnexpectedEscapeSequence": "Unexpected escape sequence continuation",
14711470
"UnexpectedField": "unexpected field '{json_field}'",
14721471
"_UnexpectedField.comment": "An example of {json_field} is identifer.",

‎src/vcpkg-test/system.process.cpp‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,66 @@ TEST_CASE ("no closes-stdout crash", "[system.process]")
6060
REQUIRE(run.exit_code == 0);
6161
REQUIRE(run.output == "hello world");
6262
}
63+
64+
TEST_CASE ("command try_append", "[system.process]")
65+
{
66+
{
67+
Command a;
68+
REQUIRE(a.try_append(Command{"b"}));
69+
REQUIRE(a.command_line() == "b");
70+
}
71+
72+
{
73+
Command a{"a"};
74+
REQUIRE(a.try_append(Command{}));
75+
REQUIRE(a.command_line() == "a");
76+
}
77+
78+
{
79+
Command a{"a"};
80+
REQUIRE(a.try_append(Command{"b"}));
81+
REQUIRE(a.command_line() == "a b");
82+
}
83+
84+
// size limits
85+
86+
std::string one_string(1, 'a');
87+
std::string big_string(Command::maximum_allowed, 'a');
88+
std::string bigger_string(Command::maximum_allowed + 1, 'a');
89+
Command empty_cmd;
90+
Command one_cmd{one_string};
91+
Command big_cmd{big_string};
92+
Command bigger_cmd{bigger_string};
93+
94+
REQUIRE(!bigger_cmd.try_append(empty_cmd));
95+
REQUIRE(bigger_cmd.command_line() == bigger_string);
96+
97+
REQUIRE(big_cmd.try_append(empty_cmd));
98+
REQUIRE(big_cmd.command_line() == big_string);
99+
100+
{
101+
auto cmd = empty_cmd;
102+
REQUIRE(!cmd.try_append(bigger_cmd));
103+
REQUIRE(cmd.empty());
104+
REQUIRE(cmd.try_append(big_cmd));
105+
REQUIRE(cmd.command_line() == big_string);
106+
}
107+
108+
{
109+
auto cmd = one_cmd;
110+
REQUIRE(!cmd.try_append(big_cmd));
111+
REQUIRE(cmd.command_line() == one_string);
112+
// does not fit due to the needed space
113+
std::string almost_string(Command::maximum_allowed - 1, 'a');
114+
Command almost_cmd{almost_string};
115+
REQUIRE(!cmd.try_append(almost_cmd));
116+
REQUIRE(cmd.command_line() == one_string);
117+
// fits exactly
118+
std::string ok_string(Command::maximum_allowed - 2, 'a');
119+
Command ok_cmd{ok_string};
120+
REQUIRE(cmd.try_append(ok_cmd));
121+
auto expected = big_string;
122+
expected[1] = ' ';
123+
REQUIRE(cmd.command_line() == expected);
124+
}
125+
}

0 commit comments

Comments
 (0)