hrev51693 adds 1 changeset to branch 'master'
old head: 11b65332b4e0d10f818d608c8f2f28322b398383
new head: 3094fef308a6aaaef9827863a0a99640bdaaa5af
overview:
http://cgit.haiku-os.org/haiku/log/?qt=range&q=3094fef308a6+%5E11b65332b4e0
----------------------------------------------------------------------------
3094fef308a6: HaikuDepot : More Backend Communications Improvements
* Further improves the logging and provides some
basic performance numbers.
* Moves the bulk-load logic out of the data-model
class.
* Introduces a state-machine for the bulk-load
process so that it will be more easily able to be
shifted to non-blocking IO when the HTTP libraries
can do that.
* Implements concurrent loading of the bulk-data to
hopefully improve lead time for icons and meta-data.
* Loads data to a temporary file and then moves to the
final location in order to avoid partially written
data in the cache.
* Handles situations where no network is available;
prevents attempt to access the network.
* Allows bulk-load processes to be cancelled when the
application quits.
* Introduces command-line arguments to help simulate
scenarios to help with testing performance and
network absence.
* Implements ordered insert and binary search in the
'List' class + basic unit test.
[ Andrew Lindesay <apl@xxxxxxxxxxxxxx> ]
----------------------------------------------------------------------------
Revision: hrev51693
Commit: 3094fef308a6aaaef9827863a0a99640bdaaa5af
URL: http://cgit.haiku-os.org/haiku/commit/?id=3094fef308a6
Author: Andrew Lindesay <apl@xxxxxxxxxxxxxx>
Date: Tue Dec 19 19:56:53 2017 UTC
----------------------------------------------------------------------------
47 files changed, 1544 insertions(+), 948 deletions(-)
src/apps/haikudepot/Jamfile | 3 +
src/apps/haikudepot/List.h | 92 ++++++
src/apps/haikudepot/model/LocalIconStore.cpp | 77 ++---
src/apps/haikudepot/model/LocalIconStore.h | 4 +-
src/apps/haikudepot/model/Logger.h | 2 -
src/apps/haikudepot/model/Model.cpp | 329 ++++++++-----------
src/apps/haikudepot/model/Model.h | 72 ++--
src/apps/haikudepot/model/PackageInfo.cpp | 29 +-
src/apps/haikudepot/model/PackageInfo.h | 4 +-
.../haikudepot/server/AbstractServerProcess.cpp | 276 ++++++++++++++--
.../haikudepot/server/AbstractServerProcess.h | 80 ++++-
.../haikudepot/server/PkgDataUpdateProcess.cpp | 266 ++++++++-------
.../haikudepot/server/PkgDataUpdateProcess.h | 26 +-
.../server/RepositoryDataUpdateProcess.cpp | 198 +++++------
.../server/RepositoryDataUpdateProcess.h | 19 +-
.../server/ServerIconExportUpdateProcess.cpp | 144 +++++++-
.../server/ServerIconExportUpdateProcess.h | 25 +-
src/apps/haikudepot/server/ServerSettings.cpp | 44 +++
src/apps/haikudepot/server/ServerSettings.h | 10 +
.../server/dumpexportpkg/DumpExportPkg.cpp | 2 +-
.../server/dumpexportpkg/DumpExportPkg.h | 2 +-
.../dumpexportpkg/DumpExportPkgCategory.cpp | 2 +-
.../server/dumpexportpkg/DumpExportPkgCategory.h | 2 +-
.../dumpexportpkg/DumpExportPkgJsonListener.cpp | 325 +++++++++---------
.../dumpexportpkg/DumpExportPkgJsonListener.h | 4 +-
.../dumpexportpkg/DumpExportPkgScreenshot.cpp | 2 +-
.../dumpexportpkg/DumpExportPkgScreenshot.h | 2 +-
.../dumpexportpkg/DumpExportPkgVersion.cpp | 2 +-
.../server/dumpexportpkg/DumpExportPkgVersion.h | 2 +-
.../DumpExportRepository.cpp | 2 +-
.../dumpexportrepository/DumpExportRepository.h | 2 +-
.../DumpExportRepositoryJsonListener.cpp | 217 ++++++------
.../DumpExportRepositoryJsonListener.h | 4 +-
.../DumpExportRepositorySource.cpp | 38 ++-
.../DumpExportRepositorySource.h | 8 +-
src/apps/haikudepot/tar/TarArchiveService.cpp | 8 +-
src/apps/haikudepot/tar/TarArchiveService.h | 4 +-
src/apps/haikudepot/ui/App.cpp | 46 ++-
src/apps/haikudepot/ui/MainWindow.cpp | 11 +-
src/apps/haikudepot/ui/MainWindow.h | 4 +
src/apps/haikudepot/util/StorageUtils.cpp | 55 +++-
src/apps/haikudepot/util/StorageUtils.h | 6 +-
.../util/ToFileUrlProtocolListener.cpp | 9 +
.../haikudepot/util/ToFileUrlProtocolListener.h | 3 +
.../DumpExportRepositoryJsonListenerTest.cpp | 6 +-
.../apps/haikudepot/HaikuDepotTestAddon.cpp | 2 +
src/tests/apps/haikudepot/Jamfile | 22 +-
----------------------------------------------------------------------------
diff --git a/src/apps/haikudepot/Jamfile b/src/apps/haikudepot/Jamfile
index f6db9f3..02a988d 100644
--- a/src/apps/haikudepot/Jamfile
+++ b/src/apps/haikudepot/Jamfile
@@ -69,6 +69,8 @@ Application HaikuDepot :
MarkupTextView.cpp
MessagePackageListener.cpp
Model.cpp
+ BulkLoadContext.cpp
+ BulkLoadStateMachine.cpp
PackageAction.cpp
PackageActionHandler.cpp
PackageContentsView.cpp
@@ -98,6 +100,7 @@ Application HaikuDepot :
# network + server
AbstractServerProcess.cpp
+ AbstractSingleFileServerProcess.cpp
ServerSettings.cpp
WebAppInterface.cpp
PkgDataUpdateProcess.cpp
diff --git a/src/apps/haikudepot/List.h b/src/apps/haikudepot/List.h
index 3c2ce08..f6abc1d 100644
--- a/src/apps/haikudepot/List.h
+++ b/src/apps/haikudepot/List.h
@@ -13,6 +13,9 @@
#include <SupportDefs.h>
+#define BINARY_SEARCH_LINEAR_THRESHOLD 4
+
+
template <typename ItemType, bool PlainOldData, uint32 BlockSize = 8>
class List {
typedef List<ItemType, PlainOldData, BlockSize> SelfType;
@@ -110,6 +113,30 @@ public:
return fCount;
}
+/*! Note that the use of this method will depend on the list being ordered.
+*/
+
+ inline int32 BinarySearch(const void* context,
+ int32 (*compareFunc)(const void* context, const ItemType& item))
+ {
+ if (fCount == 0)
+ return -1;
+
+ return _BinarySearchBounded(context, compareFunc, 0, fCount -
1);
+ }
+
+ inline bool AddOrdered(const ItemType& copyFrom,
+ int32 (*compareFunc)(const ItemType& one, const ItemType& two))
+ {
+ // special case
+ if (fCount == 0
+ || compareFunc(copyFrom, ItemAtFast(fCount - 1)) > 0) {
+ return Add(copyFrom);
+ }
+
+ return _AddOrderedBounded(copyFrom, compareFunc, 0, fCount - 1);
+ }
+
inline bool Add(const ItemType& copyFrom)
{
if (_Resize(fCount + 1)) {
@@ -229,6 +256,71 @@ public:
}
private:
+ inline int32 _BinarySearchLinearBounded(
+ const void* context,
+ int32 (*compareFunc)(const void* context, const ItemType& item),
+ int32 start, int32 end)
+ {
+ for(int32 i = start; i <= end; i++) {
+ if (compareFunc(context, ItemAtFast(i)) == 0)
+ return i;
+ }
+
+ return -1;
+ }
+
+ inline int32 _BinarySearchBounded(
+ const void* context,
+ int32 (*compareFunc)(const void* context, const ItemType& item),
+ int32 start, int32 end)
+ {
+ if (end - start < BINARY_SEARCH_LINEAR_THRESHOLD)
+ return _BinarySearchLinearBounded(context, compareFunc,
start, end);
+
+ int32 mid = start + (end - start);
+
+ if (compareFunc(context, ItemAtFast(mid)) >= 0)
+ return _BinarySearchBounded(context, compareFunc, mid,
end);
+ return _BinarySearchBounded(context, compareFunc, start, mid -
1);
+ }
+
+ inline bool _AddOrderedLinearBounded(
+ const ItemType& copyFrom,
+ int32 (*compareFunc)(const ItemType& one, const ItemType& two),
+ int32 start, int32 end)
+ {
+ for(int32 i = start; i <= (end + 1); i++) {
+ bool greaterBefore = (i == start)
+ || (compareFunc(copyFrom, ItemAtFast(i - 1)) >
0);
+
+ if (greaterBefore) {
+ bool lessAfter = (i == end + 1)
+ || (compareFunc(copyFrom,
ItemAtFast(i)) <= 0);
+
+ if (lessAfter)
+ return Add(copyFrom, i);
+ }
+ }
+
+ printf("illegal state; unable to insert item into list\n");
+ exit(EXIT_FAILURE);
+ }
+
+ inline bool _AddOrderedBounded(
+ const ItemType& copyFrom,
+ int32 (*compareFunc)(const ItemType& one, const ItemType& two),
+ int32 start, int32 end)
+ {
+ if(end - start < BINARY_SEARCH_LINEAR_THRESHOLD)
+ return _AddOrderedLinearBounded(copyFrom, compareFunc,
start, end);
+
+ int32 mid = start + (end - start);
+
+ if (compareFunc(copyFrom, ItemAtFast(mid)) >= 0)
+ return _AddOrderedBounded(copyFrom, compareFunc, mid,
end);
+ return _AddOrderedBounded(copyFrom, compareFunc, start, mid -
1);
+ }
+
inline bool _Resize(uint32 count)
{
if (count > fAllocatedCount) {
diff --git a/src/apps/haikudepot/model/LocalIconStore.cpp
b/src/apps/haikudepot/model/LocalIconStore.cpp
index c01bfd1..5cc8823 100644
--- a/src/apps/haikudepot/model/LocalIconStore.cpp
+++ b/src/apps/haikudepot/model/LocalIconStore.cpp
@@ -15,10 +15,9 @@
#include "StorageUtils.h"
-LocalIconStore::LocalIconStore()
+LocalIconStore::LocalIconStore(const BPath& path)
{
- if (_EnsureIconStoragePath(fIconStoragePath) != B_OK)
- fprintf(stdout, "unable to setup icon storage\n");
+ fIconStoragePath = path;
}
@@ -42,62 +41,26 @@ LocalIconStore::_HasIconStoragePath() const
status_t
LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const
{
- if (_HasIconStoragePath()) {
- BPath bestIconPath;
- BPath iconPkgPath(fIconStoragePath);
- bool exists;
- bool isDir;
-
- if ( (iconPkgPath.Append("hicn") == B_OK)
- && (iconPkgPath.Append(pkgName) == B_OK)
- && (StorageUtils::ExistsDirectory(iconPkgPath, &exists,
&isDir)
- == B_OK)
- && exists
- && isDir
- && (_IdentifyBestIconFileAtDirectory(iconPkgPath,
bestIconPath)
- == B_OK)
- ) {
- path = bestIconPath;
- return B_OK;
- }
- }
-
- path.Unset();
- return B_FILE_NOT_FOUND;
-}
-
-
-void
-LocalIconStore::UpdateFromServerIfNecessary() const
-{
- if (_HasIconStoragePath()) {
- BPath iconStoragePath(fIconStoragePath);
- ServerIconExportUpdateProcess service(iconStoragePath);
- service.Run();
-
- if (Logger::IsDebugEnabled()) {
- printf("did update the icons from server\n");
- }
- }
-}
-
-
-status_t
-LocalIconStore::_EnsureIconStoragePath(BPath& path) const
-{
- BPath iconStoragePath;
-
- if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK
- && iconStoragePath.Append("HaikuDepot") == B_OK
- && iconStoragePath.Append("__allicons") == B_OK
- && create_directory(iconStoragePath.Path(), 0777) == B_OK) {
- path.SetTo(iconStoragePath.Path());
+ BPath bestIconPath;
+ BPath iconPkgPath(fIconStoragePath);
+ bool exists;
+ bool isDir;
+
+ if ( (iconPkgPath.Append("hicn") == B_OK)
+ && (iconPkgPath.Append(pkgName) == B_OK)
+ && (StorageUtils::ExistsObject(iconPkgPath, &exists, &isDir,
NULL)
+ == B_OK)
+ && exists
+ && isDir
+ && (_IdentifyBestIconFileAtDirectory(iconPkgPath, bestIconPath)
+ == B_OK)
+ ) {
+ path = bestIconPath;
return B_OK;
}
path.Unset();
- fprintf(stdout, "unable to find the user cache directory for icons");
- return B_ERROR;
+ return B_FILE_NOT_FOUND;
}
@@ -121,8 +84,8 @@ LocalIconStore::_IdentifyBestIconFileAtDirectory(const
BPath& directory,
bool isDir;
if ( (workingPath.Append(iconLeafname) == B_OK
- && StorageUtils::ExistsDirectory(
- workingPath, &exists, &isDir) == B_OK)
+ && StorageUtils::ExistsObject(
+ workingPath, &exists, &isDir, NULL) == B_OK)
&& exists
&& !isDir) {
bestIconPath.SetTo(workingPath.Path());
diff --git a/src/apps/haikudepot/model/LocalIconStore.h
b/src/apps/haikudepot/model/LocalIconStore.h
index 350455a..5745d21 100644
--- a/src/apps/haikudepot/model/LocalIconStore.h
+++ b/src/apps/haikudepot/model/LocalIconStore.h
@@ -16,15 +16,13 @@
class LocalIconStore {
public:
-
LocalIconStore();
+
LocalIconStore(const BPath& path);
virtual ~LocalIconStore();
status_t TryFindIconPath(const
BString& pkgName,
BPath&
path) const;
- void
UpdateFromServerIfNecessary() const;
private:
bool _HasIconStoragePath()
const;
- status_t
_EnsureIconStoragePath(BPath& path) const;
status_t
_IdentifyBestIconFileAtDirectory(
const
BPath& directory,
BPath&
bestIconPath) const;
diff --git a/src/apps/haikudepot/model/Logger.h
b/src/apps/haikudepot/model/Logger.h
index 33cfa13..95237cd 100644
--- a/src/apps/haikudepot/model/Logger.h
+++ b/src/apps/haikudepot/model/Logger.h
@@ -2,8 +2,6 @@
* Copyright 2017, Andrew Lindesay <apl@xxxxxxxxxxxxxx>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
-
-
#ifndef LOGGER_H
#define LOGGER_H
diff --git a/src/apps/haikudepot/model/Model.cpp
b/src/apps/haikudepot/model/Model.cpp
index 2358646..4093cea 100644
--- a/src/apps/haikudepot/model/Model.cpp
+++ b/src/apps/haikudepot/model/Model.cpp
@@ -23,8 +23,6 @@
#include <Path.h>
#include "Logger.h"
-#include "PkgDataUpdateProcess.h"
-#include "RepositoryDataUpdateProcess.h"
#include "StorageUtils.h"
@@ -359,10 +357,7 @@ Model::Model()
fShowAvailablePackages(true),
fShowInstalledPackages(true),
fShowSourcePackages(false),
- fShowDevelopPackages(false),
-
- fPopulateAllPackagesThread(-1),
- fStopPopulatingAllPackages(false)
+ fShowDevelopPackages(false)
{
_UpdateIsFeaturedFilter();
@@ -417,7 +412,6 @@ Model::Model()
Model::~Model()
{
- StopPopulatingAllPackages();
}
@@ -701,7 +695,6 @@ Model::PopulatePackage(const PackageInfoRef& package,
uint32 flags)
BMessage item;
if (items.FindMessage(name, &item) !=
B_OK)
break;
-// item.PrintToStream();
BString user;
BMessage userInfo;
@@ -769,32 +762,6 @@ Model::PopulatePackage(const PackageInfoRef& package,
uint32 flags)
void
-Model::PopulateAllPackages()
-{
- StopPopulatingAllPackages();
-
- fStopPopulatingAllPackages = false;
-
- fPopulateAllPackagesThread = spawn_thread(&_PopulateAllPackagesEntry,
- "Package populator", B_NORMAL_PRIORITY, this);
- if (fPopulateAllPackagesThread >= 0)
- resume_thread(fPopulateAllPackagesThread);
-}
-
-
-void
-Model::StopPopulatingAllPackages()
-{
- if (fPopulateAllPackagesThread < 0)
- return;
-
- fStopPopulatingAllPackages = true;
- wait_for_thread(fPopulateAllPackagesThread, NULL);
- fPopulateAllPackagesThread = -1;
-}
-
-
-void
Model::SetUsername(BString username)
{
BString password;
@@ -837,8 +804,6 @@ Model::SetAuthorization(const BString& username, const
BString& password,
}
-// #pragma mark - private
-
/*! When bulk repository data comes down from the server, it will
arrive as a json.gz payload. This is stored locally as a cache
and this method will provide the on-disk storage location for
@@ -846,7 +811,7 @@ Model::SetAuthorization(const BString& username, const
BString& password,
*/
status_t
-Model::_DumpExportRepositoryDataPath(BPath& path) const
+Model::DumpExportRepositoryDataPath(BPath& path) const
{
BPath repoDataPath;
@@ -866,7 +831,26 @@ Model::_DumpExportRepositoryDataPath(BPath& path) const
status_t
-Model::_DumpExportPkgDataPath(BPath& path,
+Model::IconStoragePath(BPath& path) const
+{
+ BPath iconStoragePath;
+
+ if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK
+ && iconStoragePath.Append("HaikuDepot") == B_OK
+ && iconStoragePath.Append("__allicons") == B_OK
+ && create_directory(iconStoragePath.Path(), 0777) == B_OK) {
+ path.SetTo(iconStoragePath.Path());
+ return B_OK;
+ }
+
+ path.Unset();
+ fprintf(stdout, "unable to find the user cache directory for icons");
+ return B_ERROR;
+}
+
+
+status_t
+Model::DumpExportPkgDataPath(BPath& path,
const BString& repositorySourceCode) const
{
BPath repoDataPath;
@@ -889,24 +873,6 @@ Model::_DumpExportPkgDataPath(BPath& path,
}
-status_t
-Model::PopulateWebAppRepositoryCodes()
-{
- status_t result = B_OK;
- BPath dataPath;
-
- result = _DumpExportRepositoryDataPath(dataPath);
-
- if (result != B_OK)
- return result;
-
- RepositoryDataUpdateProcess process(dataPath, &fDepots);
- result = process.Run();
-
- return result;
-}
-
-
void
Model::_UpdateIsFeaturedFilter()
{
@@ -917,141 +883,6 @@ Model::_UpdateIsFeaturedFilter()
}
-int32
-Model::_PopulateAllPackagesEntry(void* cookie)
-{
- Model* model = static_cast<Model*>(cookie);
- model->_PopulateAllPackagesThread();
- model->_PopulateAllPackagesIcons();
- return 0;
-}
-
-
-void
-Model::_PopulateAllPackagesIcons()
-{
- fLocalIconStore.UpdateFromServerIfNecessary();
-
- int32 depotIndex = 0;
- int32 packageIndex = 0;
- int32 countIconsSet = 0;
-
- fprintf(stdout, "will populate all packages' icons\n");
-
- while (true) {
- PackageInfoRef package;
- BAutolock locker(&fLock);
-
- if (depotIndex > fDepots.CountItems()) {
- fprintf(stdout, "did populate %" B_PRId32 " packages'
icons\n",
- countIconsSet);
- return;
- }
-
- const DepotInfo& depot = fDepots.ItemAt(depotIndex);
- const PackageList& packages = depot.Packages();
-
- if (packageIndex >= packages.CountItems()) {
- // Need the next depot
- packageIndex = 0;
- depotIndex++;
- } else {
- package = packages.ItemAt(packageIndex);
-
- if (Logger::IsDebugEnabled()) {
- fprintf(stdout, "will populate package icon for
[%s]\n",
- package->Name().String());
- }
-
- if (_PopulatePackageIcon(package) == B_OK)
- countIconsSet++;
-
- packageIndex++;
- }
- }
-}
-
-
-void
-Model::_PopulatePackagesForDepot(const DepotInfo& depotInfo)
-{
- BString repositorySourceCode = depotInfo.WebAppRepositorySourceCode();
- BPath repositorySourcePkgDataPath;
-
- if (B_OK != _DumpExportPkgDataPath(repositorySourcePkgDataPath,
- repositorySourceCode)) {
- printf("unable to obtain the path for storing data for [%s]\n",
- repositorySourceCode.String());
- } else {
-
- printf("will fetch and process data for repository source
[%s]\n",
- repositorySourceCode.String());
-
- PkgDataUpdateProcess process(
- repositorySourcePkgDataPath,
- fLock, repositorySourceCode, fPreferredLanguage,
- depotInfo.Packages(),
- fCategories);
-
- process.Run();
-
- printf("did fetch and process data for repository source
[%s]\n",
- repositorySourceCode.String());
- }
-}
-
-
-void
-Model::_PopulateAllPackagesThread()
-{
- int32 depotIndex = 0;
- int32 count = 0;
-
- for (depotIndex = 0;
- depotIndex < fDepots.CountItems() &&
!fStopPopulatingAllPackages;
- depotIndex++) {
- const DepotInfo& depot = fDepots.ItemAt(depotIndex);
-
- if (depot.WebAppRepositorySourceCode().Length() > 0) {
- _PopulatePackagesForDepot(depot);
- count++;
- }
- }
-
- printf("did populate package data for %" B_PRIi32 " depots\n", count);
-}
-
-
-status_t
-Model::_PopulatePackageIcon(const PackageInfoRef& package)
-{
- BPath bestIconPath;
-
- if ( fLocalIconStore.TryFindIconPath(
- package->Name(), bestIconPath) == B_OK) {
-
- BFile bestIconFile(bestIconPath.Path(), O_RDONLY);
- BitmapRef
bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true);
- BAutolock locker(&fLock);
- package->SetIcon(bitmapRef);
-
- if (Logger::IsDebugEnabled()) {
- fprintf(stdout, "have set the package icon for [%s]
from [%s]\n",
- package->Name().String(), bestIconPath.Path());
- }
-
- return B_OK;
- }
-
- if (Logger::IsDebugEnabled()) {
- fprintf(stdout, "did not set the package icon for [%s]; no
data\n",
- package->Name().String());
- }
-
- return B_FILE_NOT_FOUND;
-}
-
-
void
Model::_PopulatePackageScreenshot(const PackageInfoRef& package,
const ScreenshotInfo& info, int32 scaledWidth, bool fromCacheOnly)
@@ -1128,3 +959,119 @@ Model::_NotifyAuthorizationChanged()
listener->AuthorizationChanged();
}
}
+
+
+// temporary - should not be required once the repo info url is used.
+static void
+normalize_repository_base_url(BUrl& url)
+{
+ if (url.Protocol() == "https")
+ url.SetProtocol("http");
+
+ BString path(url.Path());
+
+ if (path.EndsWith("/"))
+ url.SetPath(path.Truncate(path.Length() - 1));
+}
+
+
+void
+Model::ForAllDepots(void (*func)(const DepotInfo& depot, void* context),
+ void* context)
+{
+ for (int32 i = 0; i < fDepots.CountItems(); i++) {
+ DepotInfo depotInfo = fDepots.ItemAtFast(i);
+ func(depotInfo, context);
+ }
+}
+
+
+// TODO; should use the repo.info url and not the base url.
+
+void
+Model::ReplaceDepotByUrl(const BString& url,
+ DepotMapper* depotMapper, void* context)
+{
+ for (int32 i = 0; i < fDepots.CountItems(); i++) {
+ DepotInfo depotInfo = fDepots.ItemAtFast(i);
+
+ BUrl url(url);
+ BUrl depotUrlNormalized(depotInfo.BaseURL());
+
+ normalize_repository_base_url(url);
+ normalize_repository_base_url(depotUrlNormalized);
+
+ if (url == depotUrlNormalized) {
+ BAutolock locker(&fLock);
+ fDepots.Replace(i, depotMapper->MapDepot(depotInfo,
context));
+ }
+ }
+}
+
+
+void
+Model::ForAllPackages(PackageConsumer* packageConsumer, void* context)
+{
+ for (int32 i = 0; i < fDepots.CountItems(); i++) {
+ DepotInfo depotInfo = fDepots.ItemAtFast(i);
+ PackageList packages = depotInfo.Packages();
+ for(int32 j = 0; j < packages.CountItems(); j++) {
+ const PackageInfoRef& packageInfoRef =
packages.ItemAtFast(j);
+
+ if (packageInfoRef != NULL) {
+ BAutolock locker(&fLock);
+ if
(!packageConsumer->ConsumePackage(packageInfoRef, context))
+ return;
+ }
+ }
+ }
+}
+
+
+void
+Model::ForPackageByNameInDepot(const BString& depotName,
+ const BString& packageName, PackageConsumer* packageConsumer, void*
context)
+{
+ int32 depotCount = fDepots.CountItems();
+
+ for (int32 i = 0; i < depotCount; i++) {
+ DepotInfo depotInfo = fDepots.ItemAtFast(i);
+
+ if (depotInfo.Name() == depotName) {
+ int32 packageIndex =
depotInfo.PackageIndexByName(packageName);
+
+ if (-1 != packageIndex) {
+ PackageList packages = depotInfo.Packages();
+ const PackageInfoRef& packageInfoRef =
+ packages.ItemAtFast(packageIndex);
+
+ BAutolock locker(&fLock);
+ packageConsumer->ConsumePackage(packageInfoRef,
+ context);
+ }
+
+ return;
+ }
+ }
+}
+
+
+void
+Model::LogDepotsWithNoWebAppRepositoryCode() const
+{
+ int32 i;
+
+ for (i = 0; i < fDepots.CountItems(); i++) {
+ const DepotInfo& depot = fDepots.ItemAt(i);
+
+ if (depot.WebAppRepositoryCode().Length() == 0) {
+ printf("depot [%s]", depot.Name().String());
+
+ if (depot.BaseURL().Length() > 0)
+ printf(" (%s)", depot.BaseURL().String());
+
+ printf(" correlates with no repository in the haiku"
+ "depot server system\n");
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/apps/haikudepot/model/Model.h
b/src/apps/haikudepot/model/Model.h
index 0394e83..fa852b1 100644
--- a/src/apps/haikudepot/model/Model.h
+++ b/src/apps/haikudepot/model/Model.h
@@ -9,7 +9,9 @@
#include <FindDirectory.h>
#include <Locker.h>
+#include "AbstractServerProcess.h"
#include "LocalIconStore.h"
+#include "BulkLoadContext.h"
#include "PackageInfo.h"
#include "WebAppInterface.h"
@@ -37,6 +39,22 @@ public:
virtual void AuthorizationChanged() = 0;
};
+
+class DepotMapper {
+public:
+ virtual DepotInfo MapDepot(const DepotInfo& depot,
+ void*
context) = 0;
+};
+
+
+class PackageConsumer {
+public:
+ virtual bool ConsumePackage(
+ const
PackageInfoRef& packageInfoRef,
+ void*
context) = 0;
+};
+
+
typedef BReference<ModelListener> ModelListenerRef;
typedef List<ModelListenerRef, false> ModelListenerList;
@@ -118,8 +136,6 @@ public:
bool ShowDevelopPackages()
const
{
return fShowDevelopPackages; }
- status_t
PopulateWebAppRepositoryCodes();
-
// Retrieve package information
static const uint32 POPULATE_CACHED_RATING = 1 << 0;
static const uint32 POPULATE_CACHED_ICON = 1 << 1;
@@ -131,8 +147,6 @@ public:
void PopulatePackage(const
PackageInfoRef& package,
uint32
flags);
- void PopulateAllPackages();
- void
StopPopulatingAllPackages();
const StringList& SupportedLanguages() const
{
return fSupportedLanguages; }
@@ -149,20 +163,41 @@ public:
const WebAppInterface& GetWebAppInterface() const
{
return fWebAppInterface; }
+ void ReplaceDepotByUrl(
+ const
BString& url,
+
DepotMapper* depotMapper,
+ void*
context);
-private:
- status_t
_DumpExportRepositoryDataPath(
- BPath&
path) const;
- status_t
_DumpExportPkgDataPath(BPath& path,
+ void ForAllDepots(
+ void
(*func)(const DepotInfo& depot,
+
void* context),
+ void*
context);
+
+ void
ForAllPackages(PackageConsumer* packageConsumer,
+ void*
context);
+
+ void ForPackageByNameInDepot(
+ const
BString& depotName,
+ const
BString& packageName,
+
PackageConsumer* packageConsumer,
+ void*
context);
+
+ status_t IconStoragePath(BPath&
path) const;
+ status_t
DumpExportRepositoryDataPath(BPath& path) const;
+ status_t
DumpExportPkgDataPath(BPath& path,
const
BString& repositorySourceCode) const;
+
+ void
LogDepotsWithNoWebAppRepositoryCode() const;
+
+private:
void
_UpdateIsFeaturedFilter();
static int32 _PopulateAllPackagesEntry(void*
cookie);
- void
_PopulateAllPackagesThread();
- void
_PopulatePackagesForDepot(
- const
DepotInfo& depotInfo);
- void
_PopulateAllPackagesIcons();
+ void
_PopulatePackageScreenshot(
+ const
PackageInfoRef& package,
+ const
ScreenshotInfo& info,
+ int32
scaledWidth, bool fromCacheOnly);
bool _GetCacheFile(BPath&
path, BFile& file,
directory_which directory,
@@ -175,14 +210,6 @@ private:
const
char* fileName,
bool
ignoreAge, time_t maxAge) const;
- status_t _PopulatePackageIcon(
- const
PackageInfoRef& package);
- void
_PopulatePackageScreenshot(
- const
PackageInfoRef& package,
- const
ScreenshotInfo& info,
- int32
scaledWidth,
- bool
fromCacheOnly);
-
void
_NotifyAuthorizationChanged();
private:
@@ -190,8 +217,6 @@ private:
DepotList fDepots;
- LocalIconStore fLocalIconStore;
-
CategoryRef fCategoryAudio;
CategoryRef fCategoryBusiness;
CategoryRef fCategoryDevelopment;
@@ -225,9 +250,6 @@ private:
bool fShowSourcePackages;
bool fShowDevelopPackages;
- thread_id
fPopulateAllPackagesThread;
- volatile bool fStopPopulatingAllPackages;
-
StringList fSupportedLanguages;
BString fPreferredLanguage;
diff --git a/src/apps/haikudepot/model/PackageInfo.cpp
b/src/apps/haikudepot/model/PackageInfo.cpp
index 94e67f9..6607a35 100644
--- a/src/apps/haikudepot/model/PackageInfo.cpp
+++ b/src/apps/haikudepot/model/PackageInfo.cpp
@@ -1,7 +1,7 @@
/*
* Copyright 2013-2014, Stephan Aßmus <superstippi@xxxxxx>.
* Copyright 2013, Rene Gollent <rene@xxxxxxxxxxx>.
- * Copyright 2016, Andrew Lindesay <apl@xxxxxxxxxxxxxx>.
+ * Copyright 2016-2017, Andrew Lindesay <apl@xxxxxxxxxxxxxx>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
@@ -1047,10 +1047,35 @@ DepotInfo::operator!=(const DepotInfo& other) const
}
+static int32 PackageCompare(const PackageInfoRef& p1, const PackageInfoRef& p2)
+{
+ return p1->Name().Compare(p2->Name());
+}
+
+
+/*! This method will insert the package into the list of packages
+ in order so that the list of packages remains in order.
+ */
+
bool
DepotInfo::AddPackage(const PackageInfoRef& package)
{
- return fPackages.Add(package);
+ return fPackages.AddOrdered(package, &PackageCompare);
+}
+
+
+static int32 PackageFixedNameCompare(const void* context,
+ const PackageInfoRef& package)
+{
+ const BString* packageName = static_cast<BString*>(context);
+ return packageName->Compare(package->Name());
+}
+
+
+int32
+DepotInfo::PackageIndexByName(const BString& packageName)
+{
+ return fPackages.BinarySearch(&packageName, &PackageFixedNameCompare);
}
diff --git a/src/apps/haikudepot/model/PackageInfo.h
b/src/apps/haikudepot/model/PackageInfo.h
index 0a754a8..99d508a 100644
--- a/src/apps/haikudepot/model/PackageInfo.h
+++ b/src/apps/haikudepot/model/PackageInfo.h
@@ -1,6 +1,6 @@
/*
* Copyright 2013-2014, Stephan Aßmus <superstippi@xxxxxx>.
- * Copyright 2016, Andrew Lindesay <apl@xxxxxxxxxxxxxx>.
+ * Copyright 2016-2017, Andrew Lindesay <apl@xxxxxxxxxxxxxx>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef PACKAGE_INFO_H
@@ -411,6 +411,8 @@ public:
bool AddPackage(const
PackageInfoRef& package);
+ int32
PackageIndexByName(const BString& packageName);
+
void SyncPackages(const
PackageList& packages);
void SetBaseURL(const
BString& baseURL);
diff --git a/src/apps/haikudepot/server/AbstractServerProcess.cpp
b/src/apps/haikudepot/server/AbstractServerProcess.cpp
index ebb20ba..5b8b77f 100644
--- a/src/apps/haikudepot/server/AbstractServerProcess.cpp
+++ b/src/apps/haikudepot/server/AbstractServerProcess.cpp
@@ -4,13 +4,15 @@
*/
#include "AbstractServerProcess.h"
+#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <AutoDeleter.h>
+#include <Autolock.h>
#include <FileIO.h>
-#include <HttpRequest.h>
#include <HttpTime.h>
+#include <Locker.h>
#include <UrlProtocolRoster.h>
#include <support/ZlibCompressionAlgorithm.h>
@@ -18,6 +20,7 @@
#include "Logger.h"
#include "ServerSettings.h"
#include "StandardMetaDataJsonEventListener.h"
+#include "StorageUtils.h"
#include "ToFileUrlProtocolListener.h"
@@ -31,6 +34,140 @@
#define TIMEOUT_MICROSECONDS 3e+7
+AbstractServerProcess::AbstractServerProcess(
+ AbstractServerProcessListener* listener, uint32 options)
+ :
+ fLock(),
+ fListener(listener),
+ fWasStopped(false),
+ fProcessState(SERVER_PROCESS_INITIAL),
+ fErrorStatus(B_OK),
+ fOptions(options),
+ fRequest(NULL)
+{
+}
+
+
+AbstractServerProcess::~AbstractServerProcess()
+{
+}
+
+
+bool
+AbstractServerProcess::HasOption(uint32 flag)
+{
+ return (fOptions & flag) == flag;
+}
+
+
+bool
+AbstractServerProcess::ShouldAttemptNetworkDownload(bool hasDataAlready)
+{
+ return
+ !HasOption(SERVER_PROCESS_NO_NETWORKING)
+ && !(HasOption(SERVER_PROCESS_PREFER_CACHE) && hasDataAlready);
+}
+
+
+status_t
+AbstractServerProcess::Run()
+{
+ {
+ BAutolock locker(&fLock);
+
+ if (ProcessState() != SERVER_PROCESS_INITIAL) {
+ printf("cannot start server process as it is not idle");
+ return B_NOT_ALLOWED;
+ }
+
+ fProcessState = SERVER_PROCESS_RUNNING;
+ }
+
+ SetErrorStatus(RunInternal());
+
+ SetProcessState(SERVER_PROCESS_COMPLETE);
+
+ // this process may be part of a larger bulk-load process and
+ // if so, the process orchestration needs to know when this
+ // process has completed.
+
+ if (fListener != NULL)
+ fListener->ServerProcessExited();
+
+ return ErrorStatus();
+}
+
+
+bool
+AbstractServerProcess::WasStopped()
+{
+ BAutolock locker(&fLock);
+ return fWasStopped;
+}
+
+
+status_t
+AbstractServerProcess::ErrorStatus()
+{
+ BAutolock locker(&fLock);
+ return fErrorStatus;
+}
+
+
+status_t
+AbstractServerProcess::Stop()
+{
+ BAutolock locker(&fLock);
+ fWasStopped = true;
+ return StopInternal();
+}
+
+
+status_t
+AbstractServerProcess::StopInternal()
+{
+ if (fRequest != NULL) {
+ return fRequest->Stop();
+ }
+
+ return B_NOT_ALLOWED;
+}
+
+
+bool
+AbstractServerProcess::IsRunning()
+{
+ return ProcessState() == SERVER_PROCESS_RUNNING;
+}
+
+
+void
+AbstractServerProcess::SetErrorStatus(status_t value)
+{
+ BAutolock locker(&fLock);
+
+ if (fErrorStatus == B_OK) {
+ fErrorStatus = value;
+ }
+}
+
+
+void
+AbstractServerProcess::SetProcessState(process_state value)
+{
+ BAutolock locker(&fLock);
+ fProcessState = value;
+}
+
+
+process_state
+AbstractServerProcess::ProcessState()
+{
+ BAutolock locker(&fLock);
+ return fProcessState;
+}
+
+
status_t
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const
{
@@ -74,8 +211,9 @@ AbstractServerProcess::IfModifiedSinceHeaderValue(BString&
headerValue,
headerValue.SetTo(modifiedHttpTime
.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
} else {
- fprintf(stderr, "unable to parse the meta-data date and time -"
- " cannot set the 'If-Modified-Since' header\n");
+ fprintf(stderr, "unable to parse the meta-data date and time
from [%s]"
+ " - cannot set the 'If-Modified-Since' header\n",
+ metaDataPath.Path());
}
return result;
@@ -163,10 +301,52 @@ AbstractServerProcess::ParseJsonFromFileWithListener(
}
+/*! In order to reduce the chance of failure half way through downloading a
+ large file, this method will download the file to a temporary file and
+ then it can rename the file to the final target file.
+*/
+
+status_t
+AbstractServerProcess::DownloadToLocalFileAtomically(
+ const BPath& targetFilePath,
+ const BUrl& url)
+{
+ BPath temporaryFilePath(tmpnam(NULL), NULL, true);
+ status_t result = DownloadToLocalFile(
+ temporaryFilePath, url, 0, 0);
+
+ // not copying if the data has not changed because the data
will be
+ // zero length. This is if the result is APP_ERR_NOT_MODIFIED.
+ if (result == B_OK) {
+
+ // if the file is zero length then assume that
something has
+ // gone wrong.
+ off_t size;
+ bool hasFile;
+
+ result = StorageUtils::ExistsObject(temporaryFilePath,
&hasFile, NULL,
+ &size);
+
+ if (result == B_OK && hasFile && size > 0) {
+ if (rename(temporaryFilePath.Path(),
targetFilePath.Path()) != 0) {
+ printf("[%s] did rename [%s] --> [%s]\n",
+ Name(), temporaryFilePath.Path(),
targetFilePath.Path());
+ result = B_IO_ERROR;
+ }
+ }
+ }
+
+ return result;
+}
+
+
status_t
AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
const BUrl& url, uint32 redirects, uint32 failures)
{
+ if (WasStopped())
+ return B_CANCELED;
+
if (redirects > MAX_REDIRECTS) {
fprintf(stdout, "exceeded %d redirects --> failure\n",
MAX_REDIRECTS);
return B_IO_ERROR;
@@ -177,10 +357,10 @@ AbstractServerProcess::DownloadToLocalFile(const BPath&
targetFilePath,
return B_IO_ERROR;
}
- fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
- targetFilePath.Path());
+ fprintf(stdout, "[%s] will stream '%s' to [%s]\n",
+ Name(), url.UrlString().String(), targetFilePath.Path());
- ToFileUrlProtocolListener listener(targetFilePath, LoggingName(),
+ ToFileUrlProtocolListener listener(targetFilePath, Name(),
Logger::IsTraceEnabled());
BHttpHeaders headers;
@@ -195,57 +375,77 @@ AbstractServerProcess::DownloadToLocalFile(const BPath&
targetFilePath,
headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
}
- BHttpRequest *request = dynamic_cast<BHttpRequest *>(
- BUrlProtocolRoster::MakeRequest(url, &listener));
- ObjectDeleter<BHttpRequest> requestDeleter(request);
- request->SetHeaders(headers);
- request->SetMaxRedirections(0);
- request->SetTimeout(TIMEOUT_MICROSECONDS);
+ thread_id thread;
+
+ {
+ fRequest = dynamic_cast<BHttpRequest *>(
+ BUrlProtocolRoster::MakeRequest(url, &listener));
+ fRequest->SetHeaders(headers);
+ fRequest->SetMaxRedirections(0);
+ fRequest->SetTimeout(TIMEOUT_MICROSECONDS);
+ thread = fRequest->Run();
+ }
- thread_id thread = request->Run();
wait_for_thread(thread, NULL);
const BHttpResult& result = dynamic_cast<const BHttpResult&>(
- request->Result());
-
+ fRequest->Result());
int32 statusCode = result.StatusCode();
+ const BHttpHeaders responseHeaders = result.Headers();
+ const char *locationC = responseHeaders["Location"];
+ BString location;
+
+ if (locationC != NULL)
+ location.SetTo(locationC);
+
+ delete fRequest;
+ fRequest = NULL;
if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
- fprintf(stdout, "did complete streaming data\n");
+ fprintf(stdout, "[%s] did complete streaming data [%"
+ B_PRIdSSIZE " bytes]\n", Name(),
listener.ContentLength());
return B_OK;
} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
- fprintf(stdout, "remote data has not changed since [%s]\n",
- ifModifiedSinceHeader.String());
+ fprintf(stdout, "[%s] remote data has not changed since [%s]\n",
+ Name(), ifModifiedSinceHeader.String());
return APP_ERR_NOT_MODIFIED;
} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
- const BHttpHeaders responseHeaders = result.Headers();
- const char *locationValue = responseHeaders["Location"];
-
- if (locationValue != NULL && strlen(locationValue) != 0) {
- BUrl location(result.Url(), locationValue);
- fprintf(stdout, "will redirect to; %s\n",
- location.UrlString().String());
- return DownloadToLocalFile(targetFilePath,
location,
- redirects + 1, 0);
+ if (location.Length() != 0) {
+ BUrl location(result.Url(), location);
+ fprintf(stdout, "[%s] will redirect to; %s\n",
+ Name(), location.UrlString().String());
+ return DownloadToLocalFile(targetFilePath, location,
+ redirects + 1, 0);
}
- fprintf(stdout, "unable to find 'Location' header for
redirect\n");
+ fprintf(stdout, "[%s] unable to find 'Location' header for
redirect\n",
+ Name());
return B_IO_ERROR;
} else {
if (statusCode == 0 || (statusCode / 100) == 5) {
- fprintf(stdout, "error response from server; %"
B_PRId32 " --> "
+ fprintf(stdout, "error response from server [%"
B_PRId32 "] --> "
"retry...\n", statusCode);
return DownloadToLocalFile(targetFilePath, url,
redirects,
failures + 1);
}
- fprintf(stdout, "unexpected response from server; %" B_PRId32
"\n",
- statusCode);
+ fprintf(stdout, "[%s] unexpected response from server [%"
B_PRId32 "]\n",
+ Name(), statusCode);
return B_IO_ERROR;
}
}
+status_t
+AbstractServerProcess::DeleteLocalFile(const BPath& currentFilePath)
+{
+ if (0 == remove(currentFilePath.Path()))
+ return B_OK;
+
+ return B_IO_ERROR;
+}
+
+
/*! When a file is damaged or corrupted in some way, the file should be
'moved
aside' so that it is not involved in the next update. This method will
create such an alternative 'damaged' file location and move this file to
@@ -263,13 +463,19 @@ AbstractServerProcess::MoveDamagedFileAside(const BPath&
currentFilePath)
damagedFilePath.Append(damagedLeaf.String());
if (0 != rename(currentFilePath.Path(), damagedFilePath.Path())) {
- printf("unable to move damaged file [%s] aside to [%s]\n",
- currentFilePath.Path(), damagedFilePath.Path());
+ printf("[%s] unable to move damaged file [%s] aside to [%s]\n",
+ Name(), currentFilePath.Path(), damagedFilePath.Path());
return B_IO_ERROR;
}
- printf("did move damaged file [%s] aside to [%s]\n",
- currentFilePath.Path(), damagedFilePath.Path());
+ printf("[%s] did move damaged file [%s] aside to [%s]\n",
+ Name(), currentFilePath.Path(), damagedFilePath.Path());
return B_OK;
}
+
+
+bool
+AbstractServerProcess::IsSuccess(status_t e) {
+ return e == B_OK || e == APP_ERR_NOT_MODIFIED;
+}
diff --git a/src/apps/haikudepot/server/AbstractServerProcess.h
b/src/apps/haikudepot/server/AbstractServerProcess.h
index de8c0d0..f8288b1 100644
--- a/src/apps/haikudepot/server/AbstractServerProcess.h
+++ b/src/apps/haikudepot/server/AbstractServerProcess.h
@@ -6,26 +6,66 @@
#ifndef ABSTRACT_SERVER_PROCESS_H
#define ABSTRACT_SERVER_PROCESS_H
+#include <HttpRequest.h>
#include <Json.h>
#include <String.h>
#include <Url.h>
#include "StandardMetaData.h"
+#include "Stoppable.h"
-#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
+#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
+#define APP_ERR_NO_DATA (B_APP_ERROR_BASE + 453)
-class AbstractServerProcess {
+typedef enum process_options {
+ SERVER_PROCESS_NO_NETWORKING = 1 << 0,
+ SERVER_PROCESS_PREFER_CACHE = 1 << 1,
+ SERVER_PROCESS_DROP_CACHE = 1 << 2
+} process_options;
+
+
+typedef enum process_state {
+ SERVER_PROCESS_INITIAL = 1,
+ SERVER_PROCESS_RUNNING = 2,
+ SERVER_PROCESS_COMPLETE = 3
+} process_state;
+
+
+/*! Clients are able to subclass from this 'interface' in order to accept
+ call-backs when a process has exited; either through success or through
+ failure.
+ */
+
+class AbstractServerProcessListener {
public:
- virtual status_t Run() = 0;
+ virtual void ServerProcessExited() = 0;
+};
+
+
+class AbstractServerProcess : public Stoppable {
+public:
+
AbstractServerProcess(
+
AbstractServerProcessListener* listener,
+ uint32
options);
+ virtual
~AbstractServerProcess();
+
+ virtual const char* Name() = 0;
+ status_t Run();
+ status_t Stop();
+ status_t ErrorStatus();
+ bool IsRunning();
+ bool WasStopped();
protected:
+ virtual status_t RunInternal() = 0;
+ virtual status_t StopInternal();
+
virtual void GetStandardMetaDataPath(
BPath&
path) const = 0;
virtual void GetStandardMetaDataJsonPath(
BString& jsonPath) const = 0;
- virtual const char* LoggingName() const = 0;
status_t
IfModifiedSinceHeaderValue(
BString& headerValue) const;
@@ -43,15 +83,41 @@ protected:
BJsonEventListener *listener,
const
BPath& path) const;
- status_t DownloadToLocalFile(
+ status_t
DownloadToLocalFileAtomically(
const
BPath& targetFilePath,
- const
BUrl& url,
- uint32
redirects, uint32 failures);
+ const
BUrl& url);
+
+ status_t DeleteLocalFile(const
BPath& currentFilePath);
status_t MoveDamagedFileAside(
const
BPath& currentFilePath);
+ bool HasOption(uint32 flag);
+ bool
ShouldAttemptNetworkDownload(
+ bool
hasDataAlready);
+
+ static bool IsSuccess(status_t e);
+
private:
+ BLocker fLock;
+ AbstractServerProcessListener*
+ fListener;
+ bool fWasStopped;
+ process_state fProcessState;
+ status_t fErrorStatus;
+ uint32 fOptions;
+
+ BHttpRequest* fRequest;
+
+ process_state ProcessState();
+ void SetErrorStatus(status_t
value);
+ void
SetProcessState(process_state value);
+
+ status_t DownloadToLocalFile(
+ const
BPath& targetFilePath,
+ const
BUrl& url,
+ uint32
redirects, uint32 failures);
+
bool LooksLikeGzip(const
char *pathStr) const;
};
diff --git a/src/apps/haikudepot/server/PkgDataUpdateProcess.cpp
b/src/apps/haikudepot/server/PkgDataUpdateProcess.cpp
index 961f8a4..c5d1b23 100644
--- a/src/apps/haikudepot/server/PkgDataUpdateProcess.cpp
+++ b/src/apps/haikudepot/server/PkgDataUpdateProcess.cpp
@@ -11,6 +11,7 @@
#include <Autolock.h>
#include <FileIO.h>
+#include <support/StopWatch.h>
#include <Url.h>
#include "Logger.h"
@@ -27,39 +28,46 @@
packages as they are parsed and processing them.
*/
-class PackageFillingPkgListener : public DumpExportPkgListener {
+class PackageFillingPkgListener :
+ public DumpExportPkgListener, public PackageConsumer {
public:
-
PackageFillingPkgListener(
- const
PackageList& packages,
- const
CategoryList& categories,
-
BLocker& lock);
+
PackageFillingPkgListener(Model *model,
+
BString& depotName, Stoppable* stoppable);
virtual
~PackageFillingPkgListener();
- virtual void Handle(DumpExportPkg* item);
+ virtual bool ConsumePackage(const
PackageInfoRef& package,
+ void
*context);
+ virtual bool Handle(DumpExportPkg* item);
virtual void Complete();
+ uint32 Count();
+
private:
int32
IndexOfPackageByName(const BString& name) const;
int32 IndexOfCategoryByName(
const
BString& name) const;
int32 IndexOfCategoryByCode(
const
BString& code) const;
- const PackageList& fPackages;
- const CategoryList& fCategories;
- BLocker& fLock;
-
+ BString fDepotName;
+ Model* fModel;
+ CategoryList fCategories;
+ Stoppable* fStoppable;
+ uint32 fCount;
+ bool fDebugEnabled;
};
-PackageFillingPkgListener::PackageFillingPkgListener(
- const PackageList& packages, const CategoryList& categories,
- BLocker& lock)
+PackageFillingPkgListener::PackageFillingPkgListener(Model* model,
+ BString& depotName, Stoppable* stoppable)
:
- fPackages(packages),
- fCategories(categories),
- fLock(lock)
+ fDepotName(depotName),
+ fModel(model),
+ fStoppable(stoppable),
+ fCount(0),
+ fDebugEnabled(Logger::IsDebugEnabled())
{
+ fCategories = model->Categories();
}
@@ -70,16 +78,17 @@ PackageFillingPkgListener::~PackageFillingPkgListener()
// TODO; performance could be improved by not needing the linear search
-int32
-PackageFillingPkgListener::IndexOfPackageByName(
+inline int32
+PackageFillingPkgListener::IndexOfCategoryByName(
const BString& name) const
{
int32 i;
+ int32 categoryCount = fCategories.CountItems();
- for (i = 0; i < fPackages.CountItems(); i++) {
- const PackageInfoRef& packageInfo = fPackages.ItemAt(i);
+ for (i = 0; i < categoryCount; i++) {
+ const CategoryRef categoryRef = fCategories.ItemAtFast(i);
- if (packageInfo->Name() == name)
+ if (categoryRef->Name() == name)
return i;
}
@@ -87,100 +96,95 @@ PackageFillingPkgListener::IndexOfPackageByName(
}
- // TODO; performance could be improved by not needing the linear search
-
-int32
-PackageFillingPkgListener::IndexOfCategoryByName(
- const BString& name) const
+bool
+PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package,
+ void *context)
{
+ DumpExportPkg* pkg = static_cast<DumpExportPkg*>(context);
int32 i;
- for (i = 0; i < fCategories.CountItems(); i++) {
- const CategoryRef categoryRef = fCategories.ItemAtFast(i);
+ if (0 != pkg->CountPkgVersions()) {
- if (categoryRef->Name() == name)
- return i;
- }
+ // this makes the assumption that the only version will
be the
+ // latest one.
- return -1;
-}
+ DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0);
+ if (!pkgVersion->TitleIsNull())
+ package->SetTitle(*(pkgVersion->Title()));
-void
-PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
-{
- BAutolock locker(fLock); // lock from the model.
- int32 packageIndex = IndexOfPackageByName(*(pkg->Name()));
+ if (!pkgVersion->SummaryIsNull())
+ package->SetShortDescription(*(pkgVersion->Summary()));
- if (packageIndex == -1) {
- printf("unable to find package data for pkg name [%s]\n",
- pkg->Name()->String());
- } else {
- int32 i;
- const PackageInfoRef& packageInfo =
fPackages.ItemAt(packageIndex);
+ if (!pkgVersion->DescriptionIsNull())
+
package->SetFullDescription(*(pkgVersion->Description()));
- if (0 != pkg->CountPkgVersions()) {
+ if (!pkgVersion->PayloadLengthIsNull())
+ package->SetSize(pkgVersion->PayloadLength());
+ }
- // this makes the assumption that the only
version will be the
- // latest one.
+ int32 countPkgCategories = pkg->CountPkgCategories();
- DumpExportPkgVersion* pkgVersion =
pkg->PkgVersionsItemAt(0);
+ for (i = 0; i < countPkgCategories; i++) {
+ BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code();
+ int categoryIndex = IndexOfCategoryByName(*(categoryCode));
- if (!pkgVersion->TitleIsNull())
- packageInfo->SetTitle(*(pkgVersion->Title()));
+ if (categoryIndex == -1) {
+ printf("unable to find the category for [%s]\n",
+ categoryCode->String());
+ } else {
+ package->AddCategory(
+ fCategories.ItemAtFast(categoryIndex));
+ }
+ }
- if (!pkgVersion->SummaryIsNull())
-
packageInfo->SetShortDescription(*(pkgVersion->Summary()));
+ if (!pkg->DerivedRatingIsNull()) {
+ RatingSummary summary;
+ summary.averageRating = pkg->DerivedRating();
+ package->SetRatingSummary(summary);
+ }
- if (!pkgVersion->DescriptionIsNull()) {
-
packageInfo->SetFullDescription(*(pkgVersion->Description()));
- }
+ if (!pkg->ProminenceOrderingIsNull())
+ package->SetProminence(pkg->ProminenceOrdering());
- if (!pkgVersion->PayloadLengthIsNull())
-
packageInfo->SetSize(pkgVersion->PayloadLength());
- }
+ if (!pkg->PkgChangelogContentIsNull())
+ package->SetChangelog(*(pkg->PkgChangelogContent()));
- for (i = 0; i < pkg->CountPkgCategories(); i++) {
- BString* categoryCode =
pkg->PkgCategoriesItemAt(i)->Code();
- int categoryIndex =
IndexOfCategoryByName(*(categoryCode));
-
- if (categoryIndex == -1) {
- printf("unable to find the category for [%s]\n",
- categoryCode->String());
- } else {
- packageInfo->AddCategory(
- fCategories.ItemAtFast(categoryIndex));
- }
- }
+ int32 countPkgScreenshots = pkg->CountPkgScreenshots();
- if (!pkg->DerivedRatingIsNull()) {
- RatingSummary summary;
- summary.averageRating = pkg->DerivedRating();
- packageInfo->SetRatingSummary(summary);
- }
+ for (i = 0; i < countPkgScreenshots; i++) {
+ DumpExportPkgScreenshot* screenshot =
pkg->PkgScreenshotsItemAt(i);
+ package->AddScreenshotInfo(ScreenshotInfo(
+ *(screenshot->Code()),
+ static_cast<int32>(screenshot->Width()),
+ static_cast<int32>(screenshot->Height()),
+ static_cast<int32>(screenshot->Length())
+ ));
+ }
- if (!pkg->ProminenceOrderingIsNull()) {
- packageInfo->SetProminence(pkg->ProminenceOrdering());
- }
+ if (fDebugEnabled) {
+ printf("did populate data for [%s] (%s)\n",
pkg->Name()->String(),
+ fDepotName.String());
+ }
- if (!pkg->PkgChangelogContentIsNull()) {
-
packageInfo->SetChangelog(*(pkg->PkgChangelogContent()));
- }
+ fCount++;
- for (i = 0; i < pkg->CountPkgScreenshots(); i++) {
- DumpExportPkgScreenshot* screenshot =
pkg->PkgScreenshotsItemAt(i);
- packageInfo->AddScreenshotInfo(ScreenshotInfo(
- *(screenshot->Code()),
- static_cast<int32>(screenshot->Width()),
- static_cast<int32>(screenshot->Height()),
- static_cast<int32>(screenshot->Length())
- ));
- }
+ return !fStoppable->WasStopped();
+}
- if (Logger::IsDebugEnabled()) {
- printf("did populate data for [%s]\n",
pkg->Name()->String());
- }
- }
+
+uint32
+PackageFillingPkgListener::Count()
+{
+ return fCount;
+}
+
+
+bool
+PackageFillingPkgListener::Handle(DumpExportPkg* pkg)
+{
+ fModel->ForPackageByNameInDepot(fDepotName, *(pkg->Name()), this, pkg);
+ return !fStoppable->WasStopped();
}
@@ -191,20 +195,22 @@ PackageFillingPkgListener::Complete()
PkgDataUpdateProcess::PkgDataUpdateProcess(
+ AbstractServerProcessListener* listener,
const BPath& localFilePath,
- BLocker& lock,
- BString repositorySourceCode,
BString naturalLanguageCode,
- const PackageList& packages,
- const CategoryList& categories)
+ BString repositorySourceCode,
+ BString depotName,
+ Model *model,
+ uint32 options)
:
+ AbstractSingleFileServerProcess(listener, options),
fLocalFilePath(localFilePath),
fNaturalLanguageCode(naturalLanguageCode),
fRepositorySourceCode(repositorySourceCode),
- fPackages(packages),
- fCategories(categories),
- fLock(lock)
+ fModel(model),
+ fDepotName(depotName)
{
+ fName.SetToFormat("PkgDataUpdateProcess<%s>", depotName.String());
}
@@ -213,53 +219,50 @@ PkgDataUpdateProcess::~PkgDataUpdateProcess()
}
-status_t
-PkgDataUpdateProcess::Run()
+const char*
+PkgDataUpdateProcess::Name()
{
- printf("will fetch packages' data\n");
+ return fName.String();
+}
+
+BString
+PkgDataUpdateProcess::UrlPathComponent()
+{
BString urlPath;
urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz",
fRepositorySourceCode.String(),
fNaturalLanguageCode.String());
+ return urlPath;
+}
- status_t result = DownloadToLocalFile(fLocalFilePath,
- ServerSettings::CreateFullUrl(urlPath),
- 0, 0);
-
- if (result == B_OK || result == APP_ERR_NOT_MODIFIED) {
- printf("did fetch packages' data\n");
-
- // now load the data in and process it.
-
- printf("will process packages' data\n");
- result = PopulateDataToDepots();
-
- switch (result) {
- case B_OK:
- printf("did process packages' data\n");
- break;
- default:
- MoveDamagedFileAside(fLocalFilePath);
- break;
- }
- }
- return result;
+BPath&
+PkgDataUpdateProcess::LocalPath()
+{
+ return fLocalFilePath;
}
status_t
-PkgDataUpdateProcess::PopulateDataToDepots()
+PkgDataUpdateProcess::ProcessLocalData()
{
+ BStopWatch watch("PkgDataUpdateProcess::ProcessLocalData", true);
+
PackageFillingPkgListener* itemListener =
- new PackageFillingPkgListener(fPackages, fCategories, fLock);
+ new PackageFillingPkgListener(fModel, fDepotName, this);
BulkContainerDumpExportPkgJsonListener* listener =
new BulkContainerDumpExportPkgJsonListener(itemListener);
status_t result = ParseJsonFromFileWithListener(listener,
fLocalFilePath);
+ if (Logger::IsInfoEnabled()) {
+ double secs = watch.ElapsedTime() / 1000000.0;
+ fprintf(stdout, "[%s] did process %" B_PRIi32 " packages' data "
+ "in (%6.3g secs)\n", Name(), itemListener->Count(),
secs);
+ }
+
if (B_OK != result)
return result;
@@ -280,10 +283,3 @@ PkgDataUpdateProcess::GetStandardMetaDataJsonPath(
{
jsonPath.SetTo("$.info");
}
-
-
-const char*
-PkgDataUpdateProcess::LoggingName() const
-{
- return "pkg-data-update";
-}
diff --git a/src/apps/haikudepot/server/PkgDataUpdateProcess.h
b/src/apps/haikudepot/server/PkgDataUpdateProcess.h
index 749783a..7d1486a 100644
--- a/src/apps/haikudepot/server/PkgDataUpdateProcess.h
+++ b/src/apps/haikudepot/server/PkgDataUpdateProcess.h
@@ -7,8 +7,9 @@
#define PACKAGE_DATA_UPDATE_PROCESS_H
-#include "AbstractServerProcess.h"
+#include "AbstractSingleFileServerProcess.h"
+#include "Model.h"
#include "PackageInfo.h"
#include <File.h>
@@ -17,34 +18,37 @@
#include <Url.h>
-class PkgDataUpdateProcess : public AbstractServerProcess {
+class PkgDataUpdateProcess : public AbstractSingleFileServerProcess {
public:
PkgDataUpdateProcess(
+
AbstractServerProcessListener* listener,
const
BPath& localFilePath,
-
BLocker& lock,
BString
naturalLanguageCode,
BString
repositorySourceCode,
- const
PackageList& packages,
- const
CategoryList& categories);
+ BString
depotName,
+ Model
*model,
+ uint32
options);
virtual ~PkgDataUpdateProcess();
- status_t Run();
+ const char* Name();
protected:
void
GetStandardMetaDataPath(BPath& path) const;
void
GetStandardMetaDataJsonPath(
BString& jsonPath) const;
- const char* LoggingName() const;
+
+ BString UrlPathComponent();
+ status_t ProcessLocalData();
+ BPath& LocalPath();
private:
- status_t PopulateDataToDepots();
BPath fLocalFilePath;
BString fNaturalLanguageCode;
BString fRepositorySourceCode;
- const PackageList& fPackages;
- const CategoryList& fCategories;
- BLocker& fLock;
+ Model* fModel;
+ BString fDepotName;
+ BString fName;
};
diff --git a/src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp
b/src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp
index 3218075..bbd0eac 100644
--- a/src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp
+++ b/src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp
@@ -14,6 +14,7 @@
#include "ServerSettings.h"
#include "StorageUtils.h"
+#include "Logger.h"
#include "DumpExportRepository.h"
#include "DumpExportRepositorySource.h"
#include "DumpExportRepositoryJsonListener.h"
@@ -25,13 +26,15 @@
from the server with the data about the depot.
*/
-class DepotMatchingRepositoryListener : public DumpExportRepositoryListener {
+class DepotMatchingRepositoryListener :
+ public DumpExportRepositoryListener, public DepotMapper {
public:
-
DepotMatchingRepositoryListener(
-
DepotList* depotList);
+
DepotMatchingRepositoryListener(Model* model,
+
Stoppable* stoppable);
virtual
~DepotMatchingRepositoryListener();
- virtual void Handle(DumpExportRepository*
item);
+ virtual DepotInfo MapDepot(const DepotInfo&
depot, void *context);
+ virtual bool Handle(DumpExportRepository*
item);
virtual void Complete();
private:
@@ -43,15 +46,17 @@ private:
int32
IndexOfUnassociatedDepotByUrl(
const
BString& url) const;
- DepotList* fDepotList;
-
+ Model* fModel;
+ Stoppable* fStoppable;
};
DepotMatchingRepositoryListener::DepotMatchingRepositoryListener(
- DepotList* depotList)
+ Model* model, Stoppable* stoppable)
+ :
+ fModel(model),
+ fStoppable(stoppable)
{
- fDepotList = depotList;
}
@@ -60,110 +65,86 @@
DepotMatchingRepositoryListener::~DepotMatchingRepositoryListener()
}
-void
-DepotMatchingRepositoryListener::NormalizeUrl(BUrl& url) const
-{
- if (url.Protocol() == "https")
- url.SetProtocol("http");
-
- BString path(url.Path());
-
- if (path.EndsWith("/"))
- url.SetPath(path.Truncate(path.Length() - 1));
-}
-
-
-bool
-DepotMatchingRepositoryListener::IsUnassociatedDepotByUrl(
- const DepotInfo& depotInfo, const BString& urlStr) const
-{
- if (depotInfo.BaseURL().Length() > 0
- && depotInfo.WebAppRepositorySourceCode().Length() == 0) {
- BUrl depotInfoBaseUrl(depotInfo.BaseURL());
- BUrl url(urlStr);
-
- NormalizeUrl(depotInfoBaseUrl);
- NormalizeUrl(url);
-
- if (depotInfoBaseUrl == url)
- return true;
- }
+struct repository_and_repository_source {
+ DumpExportRepository* repository;
+ DumpExportRepositorySource* repositorySource;
+};
- return false;
-}
+/*! This is invoked as a result of logic in 'Handle(..)' that requests that the
+ model call this method with the requested DepotInfo instance.
+*/
-int32
-DepotMatchingRepositoryListener::IndexOfUnassociatedDepotByUrl(
- const BString& url) const
+DepotInfo
+DepotMatchingRepositoryListener::MapDepot(const DepotInfo& depot, void
*context)
{
- int32 i;
-
- for (i = 0; i < fDepotList->CountItems(); i++) {
- const DepotInfo& depot = fDepotList->ItemAt(i);
-
- if (IsUnassociatedDepotByUrl(depot, url))
- return i;
+ repository_and_repository_source* repositoryAndRepositorySource =
+ (repository_and_repository_source*) context;
+ BString* repositoryCode =
+ repositoryAndRepositorySource->repository->Code();
+ BString* repositorySourceCode =
+ repositoryAndRepositorySource->repositorySource->Code();
+
+ DepotInfo modifiedDepotInfo(depot);
+ modifiedDepotInfo.SetWebAppRepositoryCode(BString(*repositoryCode));
+ modifiedDepotInfo.SetWebAppRepositorySourceCode(
+ BString(*repositorySourceCode));
+
+ if (Logger::IsDebugEnabled()) {
+ printf("associated dept [%s] (%s) with server repository "
+ "source [%s] (%s)\n", modifiedDepotInfo.Name().String(),
+ modifiedDepotInfo.BaseURL().String(),
+ repositorySourceCode->String(),
+
repositoryAndRepositorySource->repositorySource->Url()->String());
+ } else {
+ printf("associated depot [%s] with server repository source
[%s]\n",
+ modifiedDepotInfo.Name().String(),
repositorySourceCode->String());
}
- return -1;
+ return modifiedDepotInfo;
}
-void
+bool
DepotMatchingRepositoryListener::Handle(DumpExportRepository* repository)
{
int32 i;
for (i = 0; i < repository->CountRepositorySources(); i++) {
- DumpExportRepositorySource *repositorySource =
+ repository_and_repository_source repositoryAndRepositorySource;
+ repositoryAndRepositorySource.repository = repository;
+ repositoryAndRepositorySource.repositorySource =
repository->RepositorySourcesItemAt(i);
- int32 depotIndex = IndexOfUnassociatedDepotByUrl(
- *(repositorySource->Url()));
-
- if (depotIndex >= 0) {
- DepotInfo
modifiedDepotInfo(fDepotList->ItemAt(depotIndex));
- modifiedDepotInfo.SetWebAppRepositoryCode(
- BString(*(repository->Code())));
- modifiedDepotInfo.SetWebAppRepositorySourceCode(
- BString(*(repositorySource->Code())));
- fDepotList->Replace(depotIndex, modifiedDepotInfo);
-
- printf("associated depot [%s] with server"
- " repository source [%s]\n",
modifiedDepotInfo.Name().String(),
- repositorySource->Code()->String());
+
+ // TODO; replace with the repo info url
+ BString* url =
repositoryAndRepositorySource.repositorySource->Url();
+
+ if (url->Length() > 0) {
+ fModel->ReplaceDepotByUrl(*url, this,
+ &repositoryAndRepositorySource);
}
}
+
+ return !fStoppable->WasStopped();
}
void
DepotMatchingRepositoryListener::Complete()
{
- int32 i;
-
- for (i = 0; i < fDepotList->CountItems(); i++) {
- const DepotInfo& depot = fDepotList->ItemAt(i);
-
- if (depot.WebAppRepositoryCode().Length() == 0) {
- printf("depot [%s]", depot.Name().String());
-
- if (depot.BaseURL().Length() > 0)
- printf(" (%s)", depot.BaseURL().String());
-
- printf(" correlates with no repository in the haiku"
- "depot server system\n");
- }
- }
}
RepositoryDataUpdateProcess::RepositoryDataUpdateProcess(
+ AbstractServerProcessListener* listener,
const BPath& localFilePath,
- DepotList* depotList)
+ Model* model,
+ uint32 options)
+ :
+ AbstractSingleFileServerProcess(listener, options),
+ fLocalFilePath(localFilePath),
+ fModel(model)
{
- fLocalFilePath = localFilePath;
- fDepotList = depotList;
}
@@ -172,46 +153,32 @@
RepositoryDataUpdateProcess::~RepositoryDataUpdateProcess()
}
-status_t
-RepositoryDataUpdateProcess::Run()
+const char*
+RepositoryDataUpdateProcess::Name()
{
- printf("will fetch repositories data\n");
-
- // TODO: add language ISO code to the path; just 'en' for now.
- status_t result = DownloadToLocalFile(fLocalFilePath,
- ServerSettings::CreateFullUrl("/__repository/all-en.json.gz"),
- 0, 0);
-
- if (result == B_OK || result == APP_ERR_NOT_MODIFIED) {
- printf("did fetch repositories data\n");
-
- // now load the data in and process it.
+ return "RepositoryDataUpdateProcess";
+}
- printf("will process repository data and match to depots\n");
- result = PopulateDataToDepots();
- switch (result) {
- case B_OK:
- printf("did process repository data and match
to depots\n");
- break;
- default:
- MoveDamagedFileAside(fLocalFilePath);
- break;
- }
+BString
+RepositoryDataUpdateProcess::UrlPathComponent()
+{
+ return BString("/__repository/all-en.json.gz");
+}
- } else {
- printf("an error has arisen downloading the repositories'
data\n");
- }
- return result;
+BPath&
+RepositoryDataUpdateProcess::LocalPath()
+{
+ return fLocalFilePath;
}
status_t
-RepositoryDataUpdateProcess::PopulateDataToDepots()
+RepositoryDataUpdateProcess::ProcessLocalData()
{
DepotMatchingRepositoryListener* itemListener =
- new DepotMatchingRepositoryListener(fDepotList);
+ new DepotMatchingRepositoryListener(fModel, this);
BulkContainerDumpExportRepositoryJsonListener* listener =
new BulkContainerDumpExportRepositoryJsonListener(itemListener);
@@ -238,10 +205,3 @@ RepositoryDataUpdateProcess::GetStandardMetaDataJsonPath(
{
jsonPath.SetTo("$.info");
}
-
-
-const char*
-RepositoryDataUpdateProcess::LoggingName() const
-{
- return "repo-data-update";
-}
\ No newline at end of file
diff --git a/src/apps/haikudepot/server/RepositoryDataUpdateProcess.h
b/src/apps/haikudepot/server/RepositoryDataUpdateProcess.h
index 1226a2a..5bc4890 100644
--- a/src/apps/haikudepot/server/RepositoryDataUpdateProcess.h
+++ b/src/apps/haikudepot/server/RepositoryDataUpdateProcess.h
@@ -7,8 +7,9 @@
#define REPOSITORY_DATA_UPDATE_PROCESS_H
-#include "AbstractServerProcess.h"
+#include "AbstractSingleFileServerProcess.h"
+#include "Model.h"
#include "PackageInfo.h"
#include <File.h>
@@ -17,27 +18,29 @@
#include <Url.h>
-class RepositoryDataUpdateProcess : public AbstractServerProcess {
+class RepositoryDataUpdateProcess : public AbstractSingleFileServerProcess {
public:
RepositoryDataUpdateProcess(
+
AbstractServerProcessListener* listener,
const
BPath& localFilePath,
-
DepotList* depotList);
+ Model*
model, uint32 options);
virtual
~RepositoryDataUpdateProcess();
- status_t Run();
+ const char* Name();
protected:
void
GetStandardMetaDataPath(BPath& path) const;
void
GetStandardMetaDataJsonPath(
BString& jsonPath) const;
- const char* LoggingName() const;
-private:
- status_t PopulateDataToDepots();
+ BString UrlPathComponent();
+ status_t ProcessLocalData();
+ BPath& LocalPath();
+private:
BPath fLocalFilePath;
- DepotList* fDepotList;
+ Model* fModel;
};
diff --git a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp
b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp
index af74cbf..5d434c9 100644
--- a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp
+++ b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp
@@ -9,9 +9,12 @@
#include <sys/stat.h>
#include <time.h>
+#include <Autolock.h>
#include <FileIO.h>
+#include <support/StopWatch.h>
#include <support/ZlibCompressionAlgorithm.h>
+#include "Logger.h"
#include "ServerSettings.h"
#include "StorageUtils.h"
#include "TarArchiveService.h"
@@ -20,9 +23,17 @@
/*! This constructor will locate the cached data in a standardized location */
ServerIconExportUpdateProcess::ServerIconExportUpdateProcess(
- const BPath& localStorageDirectoryPath)
+ AbstractServerProcessListener* listener,
+ const BPath& localStorageDirectoryPath,
+ Model* model,
+ uint32 options)
+ :
+ AbstractServerProcess(listener, options),
+ fLocalStorageDirectoryPath(localStorageDirectoryPath),
+ fModel(model),
+ fLocalIconStore(LocalIconStore(localStorageDirectoryPath)),
+ fCountIconsSet(0)
{
- fLocalStorageDirectoryPath = localStorageDirectoryPath;
}
@@ -31,8 +42,106 @@
ServerIconExportUpdateProcess::~ServerIconExportUpdateProcess()
}
+const char*
+ServerIconExportUpdateProcess::Name()
+{
+ return "ServerIconExportUpdateProcess";
+}
+
+
+status_t
+ServerIconExportUpdateProcess::RunInternal()
+{
+ status_t result = B_OK;
+
+ if (IsSuccess(result) && HasOption(SERVER_PROCESS_DROP_CACHE)) {
+ result = StorageUtils::RemoveDirectoryContents(
+ fLocalStorageDirectoryPath);
+ }
+
+ if (IsSuccess(result)) {
+ bool hasData;
+
+ result = HasLocalData(&hasData);
+
+ if (IsSuccess(result) && ShouldAttemptNetworkDownload(hasData))
+ result = DownloadAndUnpack();
+
+ if (IsSuccess(result)) {
+ status_t hasDataResult = HasLocalData(&hasData);
+
+ if (IsSuccess(hasDataResult) && !hasData)
+ result = APP_ERR_NO_DATA;
+ }
+ }
+
+ if (IsSuccess(result) && !WasStopped())
+ result = Populate();
+
+ return result;
+}
+
+
+status_t
+ServerIconExportUpdateProcess::PopulateForPkg(const PackageInfoRef& package)
+{
+ BPath bestIconPath;
+
+ if ( fLocalIconStore.TryFindIconPath(
+ package->Name(), bestIconPath) == B_OK) {
+
+ BFile bestIconFile(bestIconPath.Path(), O_RDONLY);
+ BitmapRef
bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true);
+ // TODO; somehow handle the locking!
+ //BAutolock locker(&fLock);
+ package->SetIcon(bitmapRef);
+
+ if (Logger::IsDebugEnabled()) {
+ fprintf(stdout, "have set the package icon for [%s]
from [%s]\n",
+ package->Name().String(), bestIconPath.Path());
+ }
+
+ fCountIconsSet++;
+
+ return B_OK;
+ }
+
+ if (Logger::IsDebugEnabled()) {
+ fprintf(stdout, "did not set the package icon for [%s]; no
data\n",
+ package->Name().String());
+ }
+
+ return B_FILE_NOT_FOUND;
+}
+
+
+bool
+ServerIconExportUpdateProcess::ConsumePackage(
+ const PackageInfoRef& packageInfoRef, void* context)
+{
+ PopulateForPkg(packageInfoRef);
+ return !WasStopped();
+}
+
+
status_t
-ServerIconExportUpdateProcess::Run()
+ServerIconExportUpdateProcess::Populate()
+{
+ BStopWatch watch("ServerIconExportUpdateProcess::Populate", true);
+ fModel->ForAllPackages(this, NULL);
+
+ if (Logger::IsInfoEnabled()) {
+ double secs = watch.ElapsedTime() / 1000000.0;
+ fprintf(stdout, "did populate %" B_PRId32 " packages' icons"
+ " (%6.3g secs)\n", fCountIconsSet, secs);
+ }
+
+ return B_OK;
+}
+
+
+status_t
+ServerIconExportUpdateProcess::DownloadAndUnpack()
{
BPath tarGzFilePath(tmpnam(NULL));
status_t result = B_OK;
@@ -59,10 +168,15 @@ ServerIconExportUpdateProcess::Run()
zlibDecompressionParameters, tarIn);
if (result == B_OK) {
+ BStopWatch
watch("ServerIconExportUpdateProcess::DownloadAndUnpack_Unpack", true);
+
result = TarArchiveService::Unpack(*tarIn,
- fLocalStorageDirectoryPath);
+ fLocalStorageDirectoryPath, NULL);
if (result == B_OK) {
+ double secs = watch.ElapsedTime() / 1000000.0;
+ fprintf(stdout, "did unpack icon tgz in (%6.3g
secs)\n", secs);
+
if (0 != remove(tarGzFilePath.Path())) {
fprintf(stdout, "unable to delete the
temporary tgz path; "
"%s\n", tarGzFilePath.Path());
@@ -79,6 +193,17 @@ ServerIconExportUpdateProcess::Run()
}
+status_t
+ServerIconExportUpdateProcess::HasLocalData(bool* result) const
+{
+ BPath path;
+ off_t size;
+ GetStandardMetaDataPath(path);
+ return StorageUtils::ExistsObject(path, result, NULL, &size)
+ && size > 0;
+}
+
+
void
ServerIconExportUpdateProcess::GetStandardMetaDataPath(BPath& path) const
{
@@ -96,16 +221,9 @@ ServerIconExportUpdateProcess::GetStandardMetaDataJsonPath(
}
-const char*
-ServerIconExportUpdateProcess::LoggingName() const
-{
- return "icon-export-update";
-}
-
-
status_t
ServerIconExportUpdateProcess::Download(BPath& tarGzFilePath)
{
- return DownloadToLocalFile(tarGzFilePath,
- ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0);
+ return DownloadToLocalFileAtomically(tarGzFilePath,
+ ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"));
}
\ No newline at end of file
diff --git a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h
b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h
index f91c806..ceb9c83 100644
--- a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h
+++ b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h
@@ -8,6 +8,8 @@
#include "AbstractServerProcess.h"
+#include "LocalIconStore.h"
+#include "Model.h"
#include <File.h>
#include <Path.h>
@@ -15,26 +17,37 @@
#include <Url.h>
-class ServerIconExportUpdateProcess : public AbstractServerProcess {
+class ServerIconExportUpdateProcess :
+ public AbstractServerProcess, public PackageConsumer {
public:
ServerIconExportUpdateProcess(
- const
BPath& localStorageDirectoryPath);
+
AbstractServerProcessListener* listener,
+ const
BPath& localStorageDirectoryPath,
+ Model*
model, uint32 options);
virtual
~ServerIconExportUpdateProcess();
- status_t Run();
+ const char* Name();
+ status_t RunInternal();
+ virtual bool ConsumePackage(
+ const
PackageInfoRef& packageInfoRef,
+ void
*context);
protected:
+ status_t PopulateForPkg(const
PackageInfoRef& package);
+ status_t Populate();
+ status_t DownloadAndUnpack();
+ status_t HasLocalData(bool*
result) const;
void
GetStandardMetaDataPath(BPath& path) const;
void
GetStandardMetaDataJsonPath(
BString& jsonPath) const;
- const char* LoggingName() const;
-
-
private:
status_t Download(BPath&
tarGzFilePath);
BPath
fLocalStorageDirectoryPath;
+ Model* fModel;
+ LocalIconStore fLocalIconStore;
+ int32 fCountIconsSet;
};
diff --git a/src/apps/haikudepot/server/ServerSettings.cpp
b/src/apps/haikudepot/server/ServerSettings.cpp
index c3009bd..043fee4 100644
--- a/src/apps/haikudepot/server/ServerSettings.cpp
+++ b/src/apps/haikudepot/server/ServerSettings.cpp
@@ -21,6 +21,9 @@
BUrl ServerSettings::sBaseUrl = BUrl(BASEURL_DEFAULT);
BString ServerSettings::sUserAgent = BString();
pthread_once_t ServerSettings::sUserAgentInitOnce = PTHREAD_ONCE_INIT;
+bool ServerSettings::sPreferCache = false;
+bool ServerSettings::sDropCache = false;
+bool ServerSettings::sForceNoNetwork = false;
status_t
@@ -102,6 +105,7 @@ ServerSettings::_GetUserAgentVersionString()
return result;
}
+
void
ServerSettings::AugmentHeaders(BHttpHeaders& headers)
{
@@ -109,3 +113,43 @@ ServerSettings::AugmentHeaders(BHttpHeaders& headers)
}
+bool
+ServerSettings::PreferCache()
+{
+ return sPreferCache;
+}
+
+
+void
+ServerSettings::SetPreferCache(bool value)
+{
+ sPreferCache = value;
+}
+
+
+bool
+ServerSettings::DropCache()
+{
+ return sDropCache;
+}
+
+
+void
+ServerSettings::SetDropCache(bool value)
+{
+ sDropCache = value;
+}
+
+
+bool
+ServerSettings::ForceNoNetwork()
+{
+ return sForceNoNetwork;
+}
[ *** diff truncated: 2336 lines dropped *** ]
