From e633ace07267579d9860a165e288fbd1ed4ba442 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 11 Jun 2013 14:53:03 +0200 Subject: [PATCH 1/7] [ticket/11603] Fix github api url and use curl with valid user agent PHPBB3-11603 --- git-tools/merge.php | 7 ++++++- git-tools/setup_github_network.php | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/git-tools/merge.php b/git-tools/merge.php index 41a96c0890..2acd2280b9 100755 --- a/git-tools/merge.php +++ b/git-tools/merge.php @@ -124,7 +124,12 @@ function get_repository_url($username, $repository, $ssh = false) function api_request($query) { - $contents = file_get_contents("http://github.com/api/v2/json/$query"); + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query"); + curl_setopt($c, CURLOPT_RETURNTRANSFER, true); + curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); + $contents = curl_exec($c); + curl_close($c); if ($contents === false) { diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index 5f2e1609a7..87ce2616e3 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -145,7 +145,13 @@ function get_repository_url($username, $repository, $ssh = false) function api_request($query) { - $contents = file_get_contents("http://github.com/api/v2/json/$query"); + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query"); + curl_setopt($c, CURLOPT_RETURNTRANSFER, true); + curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); + $contents = curl_exec($c); + curl_close($c); + if ($contents === false) { return false; From bcc98ae3e7dce9c1c967622d2fccebbb0f07e9fe Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 12 Jun 2013 17:35:39 +0200 Subject: [PATCH 2/7] [ticket/11603] Rename network to forks and fix handling PHPBB3-11603 --- git-tools/setup_github_network.php | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index 87ce2616e3..95f8c86ba3 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -22,7 +22,7 @@ function show_usage() echo " collaborators Repositories of people who have push access to the specified repository\n"; echo " contributors Repositories of people who have contributed to the specified repository\n"; echo " organisation Repositories of members of the organisation at github\n"; - echo " network All repositories of the whole github network\n"; + echo " forks All repositories of the whole github network\n"; echo "\n"; echo "Options:\n"; @@ -55,10 +55,10 @@ exit(work($scope, $username, $repository, $developer)); function work($scope, $username, $repository, $developer) { // Get some basic data - $network = get_network($username, $repository); + $forks = get_forks($username, $repository); $collaborators = get_collaborators($username, $repository); - if ($network === false || $collaborators === false) + if ($forks === false || $collaborators === false) { echo "Error: failed to retrieve network or collaborators\n"; return 1; @@ -67,19 +67,19 @@ function work($scope, $username, $repository, $developer) switch ($scope) { case 'collaborators': - $remotes = array_intersect_key($network, $collaborators); + $remotes = array_intersect_key($forks, $collaborators); break; case 'organisation': - $remotes = array_intersect_key($network, get_organisation_members($username)); + $remotes = array_intersect_key($forks, get_organisation_members($username)); break; case 'contributors': - $remotes = array_intersect_key($network, get_contributors($username, $repository)); + $remotes = array_intersect_key($forks, get_contributors($username, $repository)); break; - case 'network': - $remotes = $network; + case 'forks': + $remotes = $forks; break; default: @@ -210,20 +210,20 @@ function get_collaborators($username, $repository) return $usernames; } -function get_network($username, $repository) +function get_forks($username, $repository) { - $request = api_request("repos/show/$username/$repository/network"); + $request = api_request("repos/$username/$repository/forks"); if ($request === false) { return false; } $usernames = array(); - foreach ($request->network as $network) + foreach ($request as $fork) { - $usernames[$network->owner] = array( - 'username' => $network->owner, - 'repository' => $network->name, + $usernames[$fork->owner->login] = array( + 'username' => $fork->owner->login, + 'repository' => $fork->name, ); } From e4ccc5e6eac2bfebb8770a1a8f73f965ef0146fc Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Thu, 13 Jun 2013 13:05:33 +0200 Subject: [PATCH 3/7] [ticket/11603] Fix github API calls - Some URLs changed - Response is a plain array now - Added error messages when API limit is reached PHPBB3-11603 --- git-tools/merge.php | 11 +++++- git-tools/setup_github_network.php | 60 ++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/git-tools/merge.php b/git-tools/merge.php index 2acd2280b9..5eb48a53f8 100755 --- a/git-tools/merge.php +++ b/git-tools/merge.php @@ -128,6 +128,7 @@ function api_request($query) curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query"); curl_setopt($c, CURLOPT_RETURNTRANSFER, true); curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); + curl_setopt($c, CURLOPT_HEADER, true); $contents = curl_exec($c); curl_close($c); @@ -135,13 +136,19 @@ function api_request($query) { throw new RuntimeException("Error: failed to retrieve pull request data\n", 4); } + $contents = json_decode($contents); - return json_decode($contents); + if (isset($contents->message) && strpos($contents->message, 'API Rate Limit') === 0) + { + exit('Reached github API Rate Limit. Please try again later' . "\n"); + } + + return $contents; } function get_pull($username, $repository, $pull_id) { - $request = api_request("pulls/$username/$repository/$pull_id"); + $request = api_request("repos/$username/$repository/pulls/$pull_id"); $pull = $request->pull; diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index 95f8c86ba3..de3ea55d13 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -60,7 +60,7 @@ function work($scope, $username, $repository, $developer) if ($forks === false || $collaborators === false) { - echo "Error: failed to retrieve network or collaborators\n"; + echo "Error: failed to retrieve forks or collaborators\n"; return 1; } @@ -143,34 +143,70 @@ function get_repository_url($username, $repository, $ssh = false) return $url_base . $username . '/' . $repository . '.git'; } -function api_request($query) +function api_request($query, $full_url = false) { $c = curl_init(); - curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query"); + if ($full_url) + { + curl_setopt($c, CURLOPT_URL, $query); + } + else + { + curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query?per_page=100"); + } curl_setopt($c, CURLOPT_RETURNTRANSFER, true); curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); + curl_setopt($c, CURLOPT_HEADER, true); $contents = curl_exec($c); curl_close($c); + $sub_request_result = array(); + if ($contents && strpos($contents, "\r\n\r\n") > 0) + { + list($header, $contents) = explode("\r\n\r\n", $contents); + foreach (explode("\n", $header) as $header_element) + { + if (strpos($header_element, 'Link') === 0) + { + list($head, $header_content) = explode(': ', $header_element); + foreach (explode(', ', $header_content) as $links) + { + list($url, $rel) = explode('; ', $links); + if ($rel == 'rel="next"') + { + $sub_request_result = api_request(substr($url, 1, -1), true); + } + } + } + } + } + if ($contents === false) { return false; } - return json_decode($contents); + $contents = json_decode($contents); + + if (isset($contents->message) && strpos($contents->message, 'API Rate Limit') === 0) + { + exit('Reached github API Rate Limit. Please try again later' . "\n"); + } + + return ($sub_request_result) ? array_merge($sub_request_result, $contents) : $contents; } function get_contributors($username, $repository) { - $request = api_request("repos/show/$username/$repository/contributors"); + $request = api_request("repos/$username/$repository/stats/contributors"); if ($request === false) { return false; } $usernames = array(); - foreach ($request->contributors as $contributor) + foreach ($request as $contribution) { - $usernames[$contributor->login] = $contributor->login; + $usernames[$contribution->author->login] = $contribution->author->login; } return $usernames; @@ -178,14 +214,14 @@ function get_contributors($username, $repository) function get_organisation_members($username) { - $request = api_request("organizations/$username/public_members"); + $request = api_request("orgs/$username/public_members"); if ($request === false) { return false; } $usernames = array(); - foreach ($request->users as $member) + foreach ($request as $member) { $usernames[$member->login] = $member->login; } @@ -195,16 +231,16 @@ function get_organisation_members($username) function get_collaborators($username, $repository) { - $request = api_request("repos/show/$username/$repository/collaborators"); + $request = api_request("repos/$username/$repository/collaborators"); if ($request === false) { return false; } $usernames = array(); - foreach ($request->collaborators as $collaborator) + foreach ($request as $collaborator) { - $usernames[$collaborator] = $collaborator; + $usernames[$collaborator->login] = $collaborator->login; } return $usernames; From 92c55ab111f5d28626975310fc93f35b3ca00ab0 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Thu, 13 Jun 2013 18:41:23 +0200 Subject: [PATCH 4/7] [ticket/11603] Fix spacing and add some comments PHPBB3-11603 --- git-tools/setup_github_network.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index de3ea55d13..3f1f61b288 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -156,17 +156,19 @@ function api_request($query, $full_url = false) } curl_setopt($c, CURLOPT_RETURNTRANSFER, true); curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); - curl_setopt($c, CURLOPT_HEADER, true); + curl_setopt($c, CURLOPT_HEADER, true); $contents = curl_exec($c); curl_close($c); $sub_request_result = array(); + // Split possible headers from the body if ($contents && strpos($contents, "\r\n\r\n") > 0) { list($header, $contents) = explode("\r\n\r\n", $contents); foreach (explode("\n", $header) as $header_element) { - if (strpos($header_element, 'Link') === 0) + // Find Link Header which gives us a link to the next page + if (strpos($header_element, 'Link: ') === 0) { list($head, $header_content) = explode(': ', $header_element); foreach (explode(', ', $header_content) as $links) @@ -174,6 +176,7 @@ function api_request($query, $full_url = false) list($url, $rel) = explode('; ', $links); if ($rel == 'rel="next"') { + // Found a next link, follow it and merge the results $sub_request_result = api_request(substr($url, 1, -1), true); } } From a7af0134c0cdd5577a094e78ddae258711591b10 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Thu, 13 Jun 2013 18:43:04 +0200 Subject: [PATCH 5/7] [ticket/11603] Split api_request into two functions (query only vs. full url) PHPBB3-11603 --- git-tools/setup_github_network.php | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index 3f1f61b288..c24968c7c2 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -143,17 +143,15 @@ function get_repository_url($username, $repository, $ssh = false) return $url_base . $username . '/' . $repository . '.git'; } -function api_request($query, $full_url = false) +function api_request($query) +{ + return api_url_request("https://api.github.com/$query?per_page=100"); +} + +function api_url_request($url) { $c = curl_init(); - if ($full_url) - { - curl_setopt($c, CURLOPT_URL, $query); - } - else - { - curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query?per_page=100"); - } + curl_setopt($c, CURLOPT_URL, $url); curl_setopt($c, CURLOPT_RETURNTRANSFER, true); curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); curl_setopt($c, CURLOPT_HEADER, true); @@ -177,7 +175,7 @@ function api_request($query, $full_url = false) if ($rel == 'rel="next"') { // Found a next link, follow it and merge the results - $sub_request_result = api_request(substr($url, 1, -1), true); + $sub_request_result = api_url_request(substr($url, 1, -1)); } } } From 1516ae7e7ed77879506a32f00c9787b95106235d Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 19 Jun 2013 13:38:03 +0200 Subject: [PATCH 6/7] [ticket/11603] Avoid using cURL PHPBB3-11603 --- git-tools/merge.php | 17 ++++++++++------- git-tools/setup_github_network.php | 21 +++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/git-tools/merge.php b/git-tools/merge.php index 5eb48a53f8..08c0ecfbd0 100755 --- a/git-tools/merge.php +++ b/git-tools/merge.php @@ -124,13 +124,16 @@ function get_repository_url($username, $repository, $ssh = false) function api_request($query) { - $c = curl_init(); - curl_setopt($c, CURLOPT_URL, "https://api.github.com/$query"); - curl_setopt($c, CURLOPT_RETURNTRANSFER, true); - curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); - curl_setopt($c, CURLOPT_HEADER, true); - $contents = curl_exec($c); - curl_close($c); + return api_url_request("https://api.github.com/$query?per_page=100"); +} + +function api_url_request($url) +{ + $contents = file_get_contents($url, false, stream_context_create(array( + 'http' => array( + 'header' => "User-Agent: phpBB/1.0\r\n", + ), + ))); if ($contents === false) { diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index c24968c7c2..e5bc89bf91 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -150,21 +150,18 @@ function api_request($query) function api_url_request($url) { - $c = curl_init(); - curl_setopt($c, CURLOPT_URL, $url); - curl_setopt($c, CURLOPT_RETURNTRANSFER, true); - curl_setopt($c, CURLOPT_USERAGENT, 'phpBB/1.0'); - curl_setopt($c, CURLOPT_HEADER, true); - $contents = curl_exec($c); - curl_close($c); + $contents = file_get_contents($url, false, stream_context_create(array( + 'http' => array( + 'header' => "User-Agent: phpBB/1.0\r\n", + ), + ))); $sub_request_result = array(); // Split possible headers from the body - if ($contents && strpos($contents, "\r\n\r\n") > 0) + if (!empty($http_response_header)) { - list($header, $contents) = explode("\r\n\r\n", $contents); - foreach (explode("\n", $header) as $header_element) - { + foreach ($http_response_header as $header_element) + { // Find Link Header which gives us a link to the next page if (strpos($header_element, 'Link: ') === 0) { @@ -179,7 +176,7 @@ function api_url_request($url) } } } - } + } } if ($contents === false) From 99e486dc8234d28aae1e9c3ed108f0b78d0033f0 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 19 Jun 2013 13:45:42 +0200 Subject: [PATCH 7/7] [ticket/11603] Throw RuntimeExceptions instead of using exit() PHPBB3-11603 --- git-tools/merge.php | 2 +- git-tools/setup_github_network.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/git-tools/merge.php b/git-tools/merge.php index 08c0ecfbd0..f6142095fb 100755 --- a/git-tools/merge.php +++ b/git-tools/merge.php @@ -143,7 +143,7 @@ function api_url_request($url) if (isset($contents->message) && strpos($contents->message, 'API Rate Limit') === 0) { - exit('Reached github API Rate Limit. Please try again later' . "\n"); + throw new RuntimeException('Reached github API Rate Limit. Please try again later' . "\n", 4); } return $contents; diff --git a/git-tools/setup_github_network.php b/git-tools/setup_github_network.php index e5bc89bf91..bc85d67ab5 100755 --- a/git-tools/setup_github_network.php +++ b/git-tools/setup_github_network.php @@ -187,7 +187,7 @@ function api_url_request($url) if (isset($contents->message) && strpos($contents->message, 'API Rate Limit') === 0) { - exit('Reached github API Rate Limit. Please try again later' . "\n"); + throw new RuntimeException('Reached github API Rate Limit. Please try again later' . "\n", 4); } return ($sub_request_result) ? array_merge($sub_request_result, $contents) : $contents;