[haiku-commits] haiku: hrev47779 - src/apps/haikudepot src/kits/shared headers/private/shared

  • From: superstippi@xxxxxx
  • To: haiku-commits@xxxxxxxxxxxxx
  • Date: Sat, 30 Aug 2014 23:39:03 +0200 (CEST)

hrev47779 adds 6 changesets to branch 'master'
old head: 7f9beaf1f8bbe631cbceb2aae9b4dc8567940b71
new head: 27dd7f6d71f889fed906e09fd1659b801675d48a
overview: http://cgit.haiku-os.org/haiku/log/?qt=range&q=27dd7f6+%5E7f9beaf

----------------------------------------------------------------------------

ffd7c61: JSON parser: Improve by using exceptions to get exact error
  
  Also strip redundant "_Parser_" prefix from private methods.

ee94259: JSON parser: Fixed increasing position after constants.
  
  The first letter is already increased by the main parse loop.

e9c77bd: JSON parser: Added TODOs.

630c253: HaikuDepot/WebAppInterface: Support bulk transfers.
  
  Refactor code to send different JSON requests.

e3fdc89: HaikuDepot: Use web app bulk transfers.
  
  Collect up to 50 packages and get information about them in a bulk transfer.
  If that fails, do it in two smaller transfers, until only one package is
  left, fall back to use the getPkg method than.
  
  Remember packages for which there is an icon on the server. After the
  first round of reading icons from the cache, only try to fetch icons from
  the server for which we know one should be there.
  
  Added quite a bit of command line output to see what is going on.

27dd7f6: HaikuDepot: Add User-Agent header to web app JSON requests.

                                      [ Stephan Aßmus <superstippi@xxxxxx> ]

----------------------------------------------------------------------------

6 files changed, 456 insertions(+), 142 deletions(-)
headers/private/shared/Json.h           |   9 +-
src/apps/haikudepot/Model.cpp           | 264 ++++++++++++++++++++++------
src/apps/haikudepot/Model.h             |  10 ++
src/apps/haikudepot/WebAppInterface.cpp | 156 +++++++++++-----
src/apps/haikudepot/WebAppInterface.h   |  12 ++
src/kits/shared/Json.cpp                | 147 +++++++++++-----

############################################################################

Commit:      ffd7c6110b379115cef52fc40e03fdcf50eecb5b
URL:         http://cgit.haiku-os.org/haiku/commit/?id=ffd7c61
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:24:40 2014 UTC

JSON parser: Improve by using exceptions to get exact error

Also strip redundant "_Parser_" prefix from private methods.

----------------------------------------------------------------------------

diff --git a/headers/private/shared/Json.h b/headers/private/shared/Json.h
index 440bf36..6df2648 100644
--- a/headers/private/shared/Json.h
+++ b/headers/private/shared/Json.h
@@ -22,10 +22,11 @@ public:
        static  status_t                        Parse(BMessage& message, 
BString& JSON);
 
 private:
-       static  BString                         _Parser_ParseString(BString& 
JSON, int32& pos);
-       static  double                          _Parser_ParseNumber(BString& 
JSON, int32& pos);
-       static  bool                            _Parser_ParseConstant(BString& 
JSON, int32& pos,
-                                                                               
                          const char* constant);
+       static  void                            _Parse(BMessage& message, 
BString& JSON);
+       static  BString                         _ParseString(BString& JSON, 
int32& pos);
+       static  double                          _ParseNumber(BString& JSON, 
int32& pos);
+       static  bool                            _ParseConstant(BString& JSON, 
int32& pos,
+                                                                        const 
char* constant);
 };
 
 } // namespace BPrivate
diff --git a/src/kits/shared/Json.cpp b/src/kits/shared/Json.cpp
index 9686266..3e4a146 100644
--- a/src/kits/shared/Json.cpp
+++ b/src/kits/shared/Json.cpp
@@ -1,5 +1,6 @@
 /*
  * Copyright 2014, Augustin Cavalier (waddlesplash)
+ * Copyright 2014, Stephan Aßmus <superstippi@xxxxxx>
  * Distributed under the terms of the MIT License.
  */
 
@@ -17,6 +18,46 @@
 
 namespace BPrivate {
 
+
+class ParseException {
+public:
+       ParseException(int32 position, BString error)
+               :
+               fPosition(position),
+               fError(error),
+               fReturnCode(B_BAD_DATA)
+       {
+       }
+
+       ParseException(int32 position, status_t returnCode)
+               :
+               fPosition(position),
+               fError(""),
+               fReturnCode(returnCode)
+       {
+       }
+
+       void PrintToStream() const {
+               const char* error;
+               if (fError.Length() > 0)
+                       error = fError.String();
+               else
+                       error = strerror(fReturnCode);
+               printf("Parse error at %ld: %s\n", fPosition, error);
+       }
+       
+       status_t ReturnCode() const
+       {
+               return fReturnCode;
+       }
+
+private:
+       int32           fPosition;
+       BString         fError;
+       status_t        fReturnCode;
+};
+
+
 status_t
 BJson::Parse(BMessage& message, const char* JSON)
 {
@@ -28,6 +69,23 @@ BJson::Parse(BMessage& message, const char* JSON)
 status_t
 BJson::Parse(BMessage& message, BString& JSON)
 {
+       try {
+               _Parse(message, JSON);
+               return B_OK;
+       } catch (ParseException e) {
+               e.PrintToStream();
+               return e.ReturnCode();
+       }
+       return B_ERROR;
+}
+
+
+// #pragma mark - Private methods
+
+
+void
+BJson::_Parse(BMessage& message, BString& JSON)
+{
        BMessageBuilder builder(message);
        int32 pos = 0;
        int32 length = JSON.Length();
@@ -59,13 +117,13 @@ BJson::Parse(BMessage& message, BString& JSON)
                        break;
 
                case '}':
-                       if (hierarchy.EndsWith("{") && (hierarchy.Length() != 
1)) {
+                       if (hierarchy.EndsWith("{") && hierarchy.Length() != 1) 
{
                                hierarchy.Truncate(hierarchy.Length() - 1);
                                builder.PopObject();
                        } else if (hierarchy.Length() == 1)
-                               return B_OK; // End of the JSON data
+                               return; // End of the JSON data
                        else
-                               return B_BAD_DATA; // Unmatched closebrace
+                               throw ParseException(pos, "Unmatched closebrace 
}");
 
             break;
 
@@ -86,78 +144,80 @@ BJson::Parse(BMessage& message, BString& JSON)
                        if (hierarchy.EndsWith("[")) {
                                hierarchy.Truncate(hierarchy.Length() - 1);
                                builder.PopObject();
-                       } else
-                               return B_BAD_DATA; // Unmatched closebracket
+                       } else {
+                               BString error("Unmatched closebrace ] 
hierarchy: ");
+                               error << hierarchy;
+                               throw ParseException(pos, error);
+                       }
 
                        break;
 
                case 't':
                {
-                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0)
-                               return B_BAD_DATA;
-                               // 'true' cannot be a key, it can only be a 
value
+                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0) {
+                               throw ParseException(pos,
+                                       "'true' cannot be a key, it can only be 
a value");
+                       }
                        
-                       if (_Parser_ParseConstant(JSON, pos, "true")) {
+                       if (_ParseConstant(JSON, pos, "true")) {
                                if (builder.What() == JSON_TYPE_ARRAY)
                                        key.SetToFormat("%" B_PRIu32, 
builder.CountNames());
                                builder.AddBool(key.String(), true);
                                key = "";
                        } else
-                               return B_BAD_DATA;
-                               // "t" out in the middle of nowhere!?
+                               throw ParseException(pos, "Unexpected 't'");
 
                        break;
                }
 
                case 'f':
                {
-                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0)
-                               return B_BAD_DATA;
-                               // 'false' cannot be a key, it can only be a 
value
+                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0) {
+                               throw ParseException(pos,
+                                       "'false' cannot be a key, it can only 
be a value");
+                       }
                        
-                       if (_Parser_ParseConstant(JSON, pos, "false")) {
+                       if (_ParseConstant(JSON, pos, "false")) {
                                if (builder.What() == JSON_TYPE_ARRAY)
                                        key.SetToFormat("%" B_PRIu32, 
builder.CountNames());
                                builder.AddBool(key.String(), false);
                                key = "";
                        } else
-                               return B_BAD_DATA;
-                               // "f" out in the middle of nowhere!?
+                               throw ParseException(pos, "Unexpected 'f'");
 
                        break;
                }
 
         case 'n':
         {
-                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0)
-                               return B_BAD_DATA;
-                               // 'null' cannot be a key, it can only be a 
value
+                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0) {
+                               throw ParseException(pos,
+                                       "'null' cannot be a key, it can only be 
a value");
+                       }
                        
-                       if (_Parser_ParseConstant(JSON, pos, "null")) {
+                       if (_ParseConstant(JSON, pos, "null")) {
                                if (builder.What() == JSON_TYPE_ARRAY)
                                        key.SetToFormat("%" B_PRIu32, 
builder.CountNames());
                                builder.AddPointer(key.String(), (void*)NULL);
                                key = "";
                        } else
-                               return B_BAD_DATA;
-                               // "n" out in the middle of nowhere!?
+                               throw ParseException(pos, "Unexpected 'n'");
 
                        break;
                }
 
                case '"':
                        if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0)
-                               key = _Parser_ParseString(JSON, pos);
+                               key = _ParseString(JSON, pos);
                        else if (builder.What() != JSON_TYPE_ARRAY && 
key.Length() > 0) {
-                               builder.AddString(key, 
_Parser_ParseString(JSON, pos));
+                               builder.AddString(key, _ParseString(JSON, pos));
                                key = "";
                        } else if (builder.What() == JSON_TYPE_ARRAY) {
                                key << builder.CountNames();
-                               builder.AddString(key, 
_Parser_ParseString(JSON, pos));
+                               builder.AddString(key, _ParseString(JSON, pos));
                                key = "";
                        } else
-                               return B_BAD_DATA;
-                               // Pretty sure it's impossible to get here, but 
you never know
+                               throw ParseException(pos, "Internal error at 
encountering \"");
 
                        break;
 
@@ -174,14 +234,15 @@ BJson::Parse(BMessage& message, BString& JSON)
                case '8':
                case '9':
                {
-                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0)
-                               return B_BAD_DATA;
-                               // Numbers cannot be keys, they can only be 
values
+                       if (builder.What() != JSON_TYPE_ARRAY && key.Length() 
== 0) {
+                               throw ParseException(pos,
+                                       "Numbers cannot be keys, they can only 
be values");
+                       }
                        
                        if (builder.What() == JSON_TYPE_ARRAY)
                                key << builder.CountNames();
 
-                       double number = _Parser_ParseNumber(JSON, pos);
+                       double number = _ParseNumber(JSON, pos);
                        builder.AddDouble(key.String(), number);
 
                        key = "";
@@ -197,16 +258,12 @@ BJson::Parse(BMessage& message, BString& JSON)
                pos++;
        }
 
-       // Reached the end of the BString without reaching the end of the 
document
-       return B_BAD_DATA;
+       throw ParseException(pos, "Unexpected end of document");
 }
 
 
-// #pragma mark - Private methods
-
-
 BString
-BJson::_Parser_ParseString(BString& JSON, int32& pos)
+BJson::_ParseString(BString& JSON, int32& pos)
 {
        if (JSON[pos] != '"') // Verify we're at the start of a string.
                return BString("");
@@ -270,7 +327,7 @@ BJson::_Parser_ParseString(BString& JSON, int32& pos)
 
 
 double
-BJson::_Parser_ParseNumber(BString& JSON, int32& pos)
+BJson::_ParseNumber(BString& JSON, int32& pos)
 {
        BString value;
 
@@ -309,7 +366,7 @@ BJson::_Parser_ParseNumber(BString& JSON, int32& pos)
 
 
 bool
-BJson::_Parser_ParseConstant(BString& JSON, int32& pos, const char* constant)
+BJson::_ParseConstant(BString& JSON, int32& pos, const char* constant)
 {
        BString value;
        JSON.CopyInto(value, pos, strlen(constant));

############################################################################

Commit:      ee942591caab3ded1222012d865605a66af17cc4
URL:         http://cgit.haiku-os.org/haiku/commit/?id=ee94259
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:25:54 2014 UTC

JSON parser: Fixed increasing position after constants.

The first letter is already increased by the main parse loop.

----------------------------------------------------------------------------

diff --git a/src/kits/shared/Json.cpp b/src/kits/shared/Json.cpp
index 3e4a146..df8506a 100644
--- a/src/kits/shared/Json.cpp
+++ b/src/kits/shared/Json.cpp
@@ -371,7 +371,9 @@ BJson::_ParseConstant(BString& JSON, int32& pos, const 
char* constant)
        BString value;
        JSON.CopyInto(value, pos, strlen(constant));
        if (value == constant) {
-               pos += strlen(constant);
+               // Increase pos by the remainder of the constant, pos will be
+               // increased for the first letter in the main parse loop.
+               pos += strlen(constant) - 1;
                return true;
        } else
                return false;

############################################################################

Commit:      e9c77bd249d56840c153bcced386ca961162e1ab
URL:         http://cgit.haiku-os.org/haiku/commit/?id=e9c77bd
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:28:55 2014 UTC

JSON parser: Added TODOs.

----------------------------------------------------------------------------

diff --git a/src/kits/shared/Json.cpp b/src/kits/shared/Json.cpp
index df8506a..a803350 100644
--- a/src/kits/shared/Json.cpp
+++ b/src/kits/shared/Json.cpp
@@ -99,6 +99,12 @@ BJson::_Parse(BMessage& message, BString& JSON)
        // in the case that we are parsing a map.
        BString key("");
        
+       // TODO: Check builder return codes and throw exception, or
+       // change builder implementation/interface to throw exceptions
+       // instead of returning errors.
+       // TODO: Elimitate more duplicated code, for example by moving
+       // more code into _ParseConstant().
+       
        while (pos < length) {
                switch (JSON[pos]) {
                case '{':

############################################################################

Commit:      630c253b03a640e3bdde8322a26fb36c11989424
URL:         http://cgit.haiku-os.org/haiku/commit/?id=630c253
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:31:27 2014 UTC

HaikuDepot/WebAppInterface: Support bulk transfers.

Refactor code to send different JSON requests.

----------------------------------------------------------------------------

diff --git a/src/apps/haikudepot/WebAppInterface.cpp 
b/src/apps/haikudepot/WebAppInterface.cpp
index fe1f017..c2a1138 100644
--- a/src/apps/haikudepot/WebAppInterface.cpp
+++ b/src/apps/haikudepot/WebAppInterface.cpp
@@ -67,6 +67,26 @@ public:
                return *this;
        }
 
+       JsonBuilder& AddStrings(const StringList& strings)
+       {
+               for (int i = 0; i < strings.CountItems(); i++)
+                       AddItem(strings.ItemAtFast(i));
+               return *this;
+       }
+
+       JsonBuilder& AddItem(const char* item)
+       {
+               if (fInList)
+                       fString << ",\"";
+               else
+                       fString << '"';
+               // TODO: Escape item
+               fString << item;
+               fString << "\"";
+               fInList = true;
+               return *this;
+       }
+
        JsonBuilder& AddValue(const char* name, const char* value)
        {
                _StartName(name);
@@ -234,25 +254,6 @@ status_t
 WebAppInterface::RetrievePackageInfo(const BString& packageName,
        BMessage& message)
 {
-       BUrl url("https://depot.haiku-os.org/api/v1/pkg";);
-       
-       ProtocolListener listener;
-       BUrlContext context;
-       BHttpHeaders headers;   
-       // Content-Type
-       headers.AddHeader("Content-Type", "application/json");
-
-       BHttpRequest request(url, true, "HTTP", &listener, &context);
-
-       // Authentication
-       if (!fUsername.IsEmpty() && !fPassword.IsEmpty()) {
-               request.SetUserName(fUsername);
-               request.SetPassword(fPassword);
-       }
-
-       request.SetMethod(B_HTTP_POST);
-       request.SetHeaders(headers);
-
        BString jsonString = JsonBuilder()
                .AddValue("jsonrpc", "2.0")
                .AddValue("id", ++fRequestIndex)
@@ -267,36 +268,41 @@ WebAppInterface::RetrievePackageInfo(const BString& 
packageName,
                .EndArray()
        .End();
 
-       printf("Sending JSON:\n%s\n", jsonString.String());
+//     printf("Sending JSON:\n%s\n", jsonString.String());
        
-       BMemoryIO* data = new BMemoryIO(
-               jsonString.String(), jsonString.Length() - 1);
-
-       request.AdoptInputData(data, jsonString.Length() - 1);
-
-       BMallocIO replyData;
-       listener.SetDownloadIO(&replyData);
-//     listener.SetDebug(true);
-
-       thread_id thread = request.Run();
-       wait_for_thread(thread, NULL);
+       return _SendJsonRequest(jsonString, message);
+}
 
-       const BHttpResult& result = dynamic_cast<const BHttpResult&>(
-               request.Result());
 
-       int32 statusCode = result.StatusCode();
-       if (statusCode != 200) {
-               printf("Response code: %" B_PRId32 "\n", statusCode);
-               return B_ERROR;
-       }
-
-       jsonString.SetTo(static_cast<const char*>(replyData.Buffer()),
-               replyData.BufferLength());
-       if (jsonString.Length() == 0)
-               return B_ERROR;
+status_t
+WebAppInterface::RetrieveBulkPackageInfo(const StringList& packageNames,
+       BMessage& message)
+{
+       BString jsonString = JsonBuilder()
+               .AddValue("jsonrpc", "2.0")
+               .AddValue("id", ++fRequestIndex)
+               .AddValue("method", "getBulkPkg")
+               .AddArray("params")
+                       .AddObject()
+                               .AddArray("pkgNames")
+                                       .AddStrings(packageNames)
+                               .EndArray()
+                               .AddValue("architectureCode", "x86_gcc2")
+                               .AddValue("naturalLanguageCode", "en")
+                               .AddValue("versionType", "LATEST")
+                               .AddArray("filter")
+                                       .AddItem("PKGCATEGORIES")
+                                       .AddItem("PKGSCREENSHOTS")
+                                       .AddItem("PKGICONS")
+                                       
.AddItem("PKGVERSIONLOCALIZATIONDESCRIPTIONS")
+                               .EndArray()
+                       .EndObject()
+               .EndArray()
+       .End();
 
-       BJson parser;
-       return parser.Parse(message, jsonString);
+//     printf("Sending JSON:\n%s\n", jsonString.String());
+       
+       return _SendJsonRequest(jsonString, message);
 }
 
 
@@ -328,3 +334,62 @@ WebAppInterface::RetrievePackageIcon(const BString& 
packageName,
 
        return B_ERROR;
 }
+
+
+status_t
+WebAppInterface::_SendJsonRequest(BString jsonString, BMessage& reply) const
+{
+       BUrl url("https://depot.haiku-os.org/api/v1/pkg";);
+       
+       ProtocolListener listener;
+       BUrlContext context;
+       BHttpHeaders headers;   
+       // Content-Type
+       headers.AddHeader("Content-Type", "application/json");
+
+       BHttpRequest request(url, true, "HTTP", &listener, &context);
+
+       // Authentication
+       if (!fUsername.IsEmpty() && !fPassword.IsEmpty()) {
+               request.SetUserName(fUsername);
+               request.SetPassword(fPassword);
+       }
+
+       request.SetMethod(B_HTTP_POST);
+       request.SetHeaders(headers);
+
+       BMemoryIO* data = new BMemoryIO(
+               jsonString.String(), jsonString.Length() - 1);
+
+       request.AdoptInputData(data, jsonString.Length() - 1);
+
+       BMallocIO replyData;
+       listener.SetDownloadIO(&replyData);
+//     listener.SetDebug(true);
+
+       thread_id thread = request.Run();
+       wait_for_thread(thread, NULL);
+
+       const BHttpResult& result = dynamic_cast<const BHttpResult&>(
+               request.Result());
+
+       int32 statusCode = result.StatusCode();
+       if (statusCode != 200) {
+               printf("Response code: %" B_PRId32 "\n", statusCode);
+               return B_ERROR;
+       }
+
+       jsonString.SetTo(static_cast<const char*>(replyData.Buffer()),
+               replyData.BufferLength());
+       if (jsonString.Length() == 0)
+               return B_ERROR;
+
+       BJson parser;
+       status_t status = parser.Parse(reply, jsonString);
+       if (status == B_BAD_DATA) {
+//             printf("Parser choked on JSON:\n%s\n", jsonString.String());
+       }
+       return status;
+}
+
+
diff --git a/src/apps/haikudepot/WebAppInterface.h 
b/src/apps/haikudepot/WebAppInterface.h
index f5e2a28..5436b99 100644
--- a/src/apps/haikudepot/WebAppInterface.h
+++ b/src/apps/haikudepot/WebAppInterface.h
@@ -9,10 +9,14 @@
 #include <Application.h>
 #include <String.h>
 
+#include "List.h"
+
 
 class BDataIO;
 class BMessage;
 
+typedef List<BString, false>   StringList;
+
 
 class WebAppInterface {
 public:
@@ -26,11 +30,19 @@ public:
                                                                        const 
BString& packageName,
                                                                        
BMessage& message);
 
+                       status_t                        RetrieveBulkPackageInfo(
+                                                                       const 
StringList& packageNames,
+                                                                       
BMessage& message);
+
                        status_t                        RetrievePackageIcon(
                                                                        const 
BString& packageName,
                                                                        
BDataIO* stream);
 
 private:
+                       status_t                        
_SendJsonRequest(BString jsonString,
+                                                                       
BMessage& reply) const;
+
+private:
                        BString                         fUsername;
                        BString                         fPassword;
        static  int                                     fRequestIndex;

############################################################################

Commit:      e3fdc8940c4621f1ad245a1a5e84f8ddb600827b
URL:         http://cgit.haiku-os.org/haiku/commit/?id=e3fdc89
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:32:37 2014 UTC

HaikuDepot: Use web app bulk transfers.

Collect up to 50 packages and get information about them in a bulk transfer.
If that fails, do it in two smaller transfers, until only one package is
left, fall back to use the getPkg method than.

Remember packages for which there is an icon on the server. After the
first round of reading icons from the cache, only try to fetch icons from
the server for which we know one should be there.

Added quite a bit of command line output to see what is going on.

----------------------------------------------------------------------------

diff --git a/src/apps/haikudepot/Model.cpp b/src/apps/haikudepot/Model.cpp
index df005cd..e995d1c 100644
--- a/src/apps/haikudepot/Model.cpp
+++ b/src/apps/haikudepot/Model.cpp
@@ -17,6 +17,7 @@
 #include <Entry.h>
 #include <FindDirectory.h>
 #include <File.h>
+#include <Message.h>
 #include <Path.h>
 
 #include "WebAppInterface.h"
@@ -598,6 +599,8 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly)
 {
        int32 depotIndex = 0;
        int32 packageIndex = 0;
+       PackageList bulkPackageList;
+       PackageList packagesWithIconsList;
 
        while (!fStopPopulatingAllPackages) {
                // Obtain PackageInfoRef while keeping the depot and package 
lists
@@ -625,16 +628,40 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly)
                if (package.Get() == NULL)
                        continue;
        
-               _PopulatePackageInfo(package, fromCacheOnly);
-               _PopulatePackageIcon(package, fromCacheOnly);
+               //_PopulatePackageInfo(package, fromCacheOnly);
+               bulkPackageList.Add(package);
+               if (bulkPackageList.CountItems() == 50) {
+                       _PopulatePackageInfos(bulkPackageList, fromCacheOnly,
+                               packagesWithIconsList);
+                       bulkPackageList.Clear();
+               }
+               if (fromCacheOnly)
+                       _PopulatePackageIcon(package, fromCacheOnly);
                // TODO: Average user rating. It needs to be shown in the
                // list view, so without the user clicking the package.
        }
+
+       if (!fStopPopulatingAllPackages && bulkPackageList.CountItems() > 0) {
+               _PopulatePackageInfos(bulkPackageList, fromCacheOnly,
+                       packagesWithIconsList);
+       }
+
+       if (!fromCacheOnly) {
+               for (int i = packagesWithIconsList.CountItems() - 1; i >= 0; 
i--) {
+                       if (fStopPopulatingAllPackages)
+                               break;
+                       const PackageInfoRef& package = 
packagesWithIconsList.ItemAtFast(i);
+                       printf("Getting/Updating native icon for %s\n",
+                               package->Title().String());
+                       _PopulatePackageIcon(package, fromCacheOnly);
+               }
+       }
 }
 
 
 void
-Model::_PopulatePackageInfo(const PackageInfoRef& package, bool fromCacheOnly)
+Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
+       PackageList& packagesWithIcons)
 {
        if (fromCacheOnly)
                return;
@@ -643,64 +670,168 @@ Model::_PopulatePackageInfo(const PackageInfoRef& 
package, bool fromCacheOnly)
        WebAppInterface interface;
        BMessage info;
 
-       status_t status = interface.RetrievePackageInfo(package->Title(), info);
+       StringList packageNames;
+       for (int i = 0; i < packages.CountItems(); i++) {
+               const PackageInfoRef& package = packages.ItemAtFast(i);
+               packageNames.Add(package->Title());
+       }
+
+       status_t status = interface.RetrieveBulkPackageInfo(packageNames, info);
        if (status == B_OK) {
                // Parse message
-               info.PrintToStream();
+//             info.PrintToStream();
                BMessage result;
-               if (info.FindMessage("result", &result) == B_OK) {
-                       BMessage categories;
-                       if (result.FindMessage("pkgCategoryCodes", &categories) 
== B_OK) {
-                               int32 index = 0;
-                               while (true) {
-                                       BString name;
-                                       name << index++;
-                                       BString category;
-                                       if (categories.FindString(name, 
&category) != B_OK)
+               BMessage pkgs;
+               if (info.FindMessage("result", &result) == B_OK
+                       && result.FindMessage("pkgs", &pkgs) == B_OK) {
+                       int32 index = 0;
+                       while (true) {
+                               BString name;
+                               name << index++;
+                               BMessage pkgInfo;
+                               if (pkgs.FindMessage(name, &pkgInfo) != B_OK)
+                                       break;
+                       
+                               BString pkgName;
+                               if (pkgInfo.FindString("name", &pkgName) != 
B_OK)
+                                       continue;
+                       
+                               // Find the PackageInfoRef
+                               bool found = false;
+                               for (int i = 0; i < packages.CountItems(); i++) 
{
+                                       const PackageInfoRef& package = 
packages.ItemAtFast(i);
+                                       if (pkgName == package->Title()) {
+                                               _PopulatePackageInfo(package, 
pkgInfo);
+                                               if (_HasNativeIcon(pkgInfo))
+                                                       
packagesWithIcons.Add(package);
+                                               packages.Remove(i);
+                                               found = true;
                                                break;
-
-                                       if (category == "AUDIO")
-                                               
package->AddCategory(CategoryAudio());
-                                       else if (category == "BUSINESS")
-                                               
package->AddCategory(CategoryBusiness());
-                                       else if (category == "DEVELOPMENT")
-                                               
package->AddCategory(CategoryDevelopment());
-                                       else if (category == "EDUCATION")
-                                               
package->AddCategory(CategoryEducation());
-                                       else if (category == "GAMES")
-                                               
package->AddCategory(CategoryGames());
-                                       else if (category == "GRAPHICS")
-                                               
package->AddCategory(CategoryGraphics());
-                                       else if (category == 
"INTERNETANDNETWORK")
-                                               
package->AddCategory(CategoryInternetAndNetwork());
-                                       else if (category == "PRODUCTIVITY")
-                                               
package->AddCategory(CategoryProductivity());
-                                       else if (category == 
"SCIENCEANDMATHEMATICS")
-                                               
package->AddCategory(CategoryScienceAndMathematics());
-                                       else if (category == 
"SYSTEMANDUTILITIES")
-                                               
package->AddCategory(CategorySystemAndUtilities());
-                                       else if (category == "VIDEO")
-                                               
package->AddCategory(CategoryVideo());
-                                       // TODO: The server will eventually 
support an API to
-                                       // get the defined categories and their 
translated names.
-                                       // This should then be used instead of 
hard-coded
-                                       // categories and translations in the 
app.
+                                       }
                                }
+                               if (!found)
+                                       printf("No matching package for %s\n", 
pkgName.String());
                        }
-                       double derivedRating;
-                       double derivedRatingSampleSize;
-                       if (result.FindDouble("derivedRating", &derivedRating) 
== B_OK
-                               && result.FindDouble("derivedRatingSampleSize",
-                                       &derivedRatingSampleSize) == B_OK) {
-                               if (derivedRatingSampleSize > 0) {
-                                       RatingSummary summary;
-                                       summary.averageRating = derivedRating;
-                                       summary.ratingCount = 
(int)derivedRatingSampleSize;
-                                       package->SetRatingSummary(summary);
-                               }
+               }
+       } else {
+               printf("Error sending request: %s\n", strerror(status));
+               int count = packages.CountItems();
+               if (count >= 4) {
+                       // Retry in smaller chunks
+                       PackageList firstHalf;
+                       PackageList secondHalf;
+                       for (int i = 0; i < count / 2; i++)
+                               firstHalf.Add(packages.ItemAtFast(i));
+                       for (int i = count / 2; i < count; i++)
+                               secondHalf.Add(packages.ItemAtFast(i));
+                       packages.Clear();
+                       _PopulatePackageInfos(firstHalf, fromCacheOnly, 
packagesWithIcons);
+                       _PopulatePackageInfos(secondHalf, fromCacheOnly, 
packagesWithIcons);
+               } else {
+                       while (packages.CountItems() > 0) {
+                               const PackageInfoRef& package = 
packages.ItemAtFast(0);
+                               _PopulatePackageInfo(package, fromCacheOnly);
+                               packages.Remove(0);
                        }
                }
        }
+
+       if (packages.CountItems() > 0) {
+               for (int i = 0; i < packages.CountItems(); i++) {
+                       const PackageInfoRef& package = packages.ItemAtFast(i);
+                       printf("No package info for %s\n", 
package->Title().String());
+               }
+       }
+}
+
+
+void
+Model::_PopulatePackageInfo(const PackageInfoRef& package, bool fromCacheOnly)
+{
+       if (fromCacheOnly)
+               return;
+       
+       // Retrieve info from web-app
+       WebAppInterface interface;
+       BMessage info;
+
+       status_t status = interface.RetrievePackageInfo(package->Title(), info);
+       if (status == B_OK) {
+               // Parse message
+//             info.PrintToStream();
+               BMessage result;
+               if (info.FindMessage("result", &result) == B_OK)
+                       _PopulatePackageInfo(package, result);
+       }
+}
+
+
+void
+Model::_PopulatePackageInfo(const PackageInfoRef& package,
+       const BMessage& data)
+{
+       const char* categoriesDebug = "";
+       const char* ratingDebug = "";
+
+       BMessage categories;
+       if (data.FindMessage("pkgCategoryCodes", &categories) == B_OK) {
+               int32 index = 0;
+               while (true) {
+                       BString name;
+                       name << index++;
+                       BString category;
+                       if (categories.FindString(name, &category) != B_OK)
+                               break;
+
+                       if (category == "AUDIO")
+                               package->AddCategory(CategoryAudio());
+                       else if (category == "BUSINESS")
+                               package->AddCategory(CategoryBusiness());
+                       else if (category == "DEVELOPMENT")
+                               package->AddCategory(CategoryDevelopment());
+                       else if (category == "EDUCATION")
+                               package->AddCategory(CategoryEducation());
+                       else if (category == "GAMES")
+                               package->AddCategory(CategoryGames());
+                       else if (category == "GRAPHICS")
+                               package->AddCategory(CategoryGraphics());
+                       else if (category == "INTERNETANDNETWORK")
+                               
package->AddCategory(CategoryInternetAndNetwork());
+                       else if (category == "PRODUCTIVITY")
+                               package->AddCategory(CategoryProductivity());
+                       else if (category == "SCIENCEANDMATHEMATICS")
+                               
package->AddCategory(CategoryScienceAndMathematics());
+                       else if (category == "SYSTEMANDUTILITIES")
+                               
package->AddCategory(CategorySystemAndUtilities());
+                       else if (category == "VIDEO")
+                               package->AddCategory(CategoryVideo());
+                       // TODO: The server will eventually support an API to
+                       // get the defined categories and their translated 
names.
+                       // This should then be used instead of hard-coded
+                       // categories and translations in the app.
+               
+                       categoriesDebug = "categories";
+               }
+       }
+       double derivedRating;
+       double derivedRatingSampleSize;
+       if (data.FindDouble("derivedRating", &derivedRating) == B_OK
+               && data.FindDouble("derivedRatingSampleSize",
+                       &derivedRatingSampleSize) == B_OK) {
+               if (derivedRatingSampleSize > 0) {
+                       RatingSummary summary;
+                       summary.averageRating = derivedRating;
+                       summary.ratingCount = (int)derivedRatingSampleSize;
+                       package->SetRatingSummary(summary);
+
+                       if (strlen(categoriesDebug) > 0)
+                               ratingDebug = ", rating";
+                       else
+                               ratingDebug = "rating";
+               }
+       }
+       printf("Populated package info for %s: %s%s\n",
+               package->Title().String(), categoriesDebug, ratingDebug);
 }
 
 
@@ -753,3 +884,32 @@ Model::_PopulatePackageIcon(const PackageInfoRef& package, 
bool fromCacheOnly)
                }
        }
 }
+
+
+bool
+Model::_HasNativeIcon(const BMessage& message) const
+{
+       BMessage pkgIcons;
+       if (message.FindMessage("pkgIcons", &pkgIcons) != B_OK)
+               return false;
+
+       if (!pkgIcons.IsEmpty())
+               pkgIcons.PrintToStream();
+
+       int32 index = 0;
+       while (true) {
+               BString name;
+               name << index++;
+               
+               BMessage typeCodeInfo;
+               if (pkgIcons.FindMessage(name, &typeCodeInfo) != B_OK)
+                       break;
+       
+               BString mediaTypeCode;
+               if (typeCodeInfo.FindString("mediaTypeCode", &mediaTypeCode) == 
B_OK
+                       && mediaTypeCode == "application/x-vnd.haiku-icon") {
+                       return true;
+               }
+       }
+       return false;
+}
diff --git a/src/apps/haikudepot/Model.h b/src/apps/haikudepot/Model.h
index 7c066a9..d9c1b55 100644
--- a/src/apps/haikudepot/Model.h
+++ b/src/apps/haikudepot/Model.h
@@ -9,6 +9,8 @@
 
 #include "PackageInfo.h"
 
+class BMessage;
+
 
 class PackageFilter : public BReferenceable {
 public:
@@ -101,12 +103,20 @@ private:
        static  int32                           _PopulateAllPackagesEntry(void* 
cookie);
                        void                            
_PopulateAllPackagesThread(bool fromCacheOnly);
 
+                       void                            _PopulatePackageInfos(
+                                                                       
PackageList& packages,
+                                                                       bool 
fromCacheOnly,
+                                                                       
PackageList& packagesWithIcons);
                        void                            _PopulatePackageInfo(
                                                                        const 
PackageInfoRef& package,
                                                                        bool 
fromCacheOnly);
+                       void                            _PopulatePackageInfo(
+                                                                       const 
PackageInfoRef& package,
+                                                                       const 
BMessage& data);
                        void                            _PopulatePackageIcon(
                                                                        const 
PackageInfoRef& package,
                                                                        bool 
fromCacheOnly);
+                       bool                            _HasNativeIcon(const 
BMessage& message) const;
 
 private:
                        BLocker                         fLock;

############################################################################

Revision:    hrev47779
Commit:      27dd7f6d71f889fed906e09fd1659b801675d48a
URL:         http://cgit.haiku-os.org/haiku/commit/?id=27dd7f6
Author:      Stephan Aßmus <superstippi@xxxxxx>
Date:        Sat Aug 30 21:40:57 2014 UTC

HaikuDepot: Add User-Agent header to web app JSON requests.

----------------------------------------------------------------------------

diff --git a/src/apps/haikudepot/WebAppInterface.cpp 
b/src/apps/haikudepot/WebAppInterface.cpp
index c2a1138..8cff1ca 100644
--- a/src/apps/haikudepot/WebAppInterface.cpp
+++ b/src/apps/haikudepot/WebAppInterface.cpp
@@ -346,6 +346,7 @@ WebAppInterface::_SendJsonRequest(BString jsonString, 
BMessage& reply) const
        BHttpHeaders headers;   
        // Content-Type
        headers.AddHeader("Content-Type", "application/json");
+       headers.AddHeader("User-Agent", "X-HDS-Client");
 
        BHttpRequest request(url, true, "HTTP", &listener, &context);
 


Other related posts:

  • » [haiku-commits] haiku: hrev47779 - src/apps/haikudepot src/kits/shared headers/private/shared - superstippi