From 58e4125be70e45bc94592f4e505579ff9df9b8b8 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Thu, 16 Feb 2023 18:23:48 +0000 Subject: [PATCH 1/2] Revert "update next major to core 13.4.1 (#6310)" This reverts commit 59764a2e416aeadfc67ed9867d3e57a22cd3e1e8. --- .github/advanced-issue-labeler.yml | 76 --- .github/auto_assign.yml | 3 - .github/workflows/Issue-Needs-Attention.yml | 3 - .github/workflows/auto-assign.yml | 13 - .github/workflows/check-changelog.yml | 21 - .github/workflows/issue-labeler.yml | 35 - .github/workflows/no-response.yml | 22 - CHANGELOG.md | 111 +--- LICENSE | 2 +- Package.swift | 3 +- dependencies.list | 6 +- evergreen/add_admin_roles.js | 13 - evergreen/config.yml | 79 +-- evergreen/install_baas.sh | 15 +- src/realm.h | 198 +----- src/realm/alloc.hpp | 2 +- src/realm/alloc_slab.cpp | 10 +- src/realm/array.hpp | 4 +- src/realm/db.cpp | 133 ++-- src/realm/db.hpp | 1 - src/realm/decimal128.cpp | 10 +- src/realm/error_codes.cpp | 26 - src/realm/error_codes.hpp | 14 - src/realm/exec/CMakeLists.txt | 6 +- src/realm/exec/realm2json.cpp | 51 +- src/realm/exec/realm_trawler.cpp | 54 +- src/realm/group.cpp | 2 +- src/realm/group.hpp | 1 - src/realm/group_writer.cpp | 6 +- src/realm/group_writer.hpp | 12 - src/realm/history.cpp | 13 +- src/realm/list.cpp | 71 +- src/realm/list.hpp | 36 +- src/realm/object-store/CMakeLists.txt | 2 + src/realm/object-store/audit.hpp | 18 +- src/realm/object-store/audit.mm | 164 ++--- .../binding_callback_thread_observer.cpp | 2 +- .../binding_callback_thread_observer.hpp | 2 +- src/realm/object-store/c_api/CMakeLists.txt | 1 - src/realm/object-store/c_api/conversion.hpp | 1 - src/realm/object-store/c_api/dictionary.cpp | 28 - .../object-store/c_api/notifications.cpp | 72 +- src/realm/object-store/c_api/realm.cpp | 7 +- src/realm/object-store/c_api/realm.hpp | 9 +- src/realm/object-store/c_api/schema.cpp | 2 +- .../object-store/c_api/socket_provider.cpp | 262 -------- src/realm/object-store/c_api/sync.cpp | 65 +- src/realm/object-store/c_api/types.hpp | 83 +-- src/realm/object-store/collection.cpp | 2 +- src/realm/object-store/collection.hpp | 2 +- src/realm/object-store/dictionary.cpp | 7 +- src/realm/object-store/dictionary.hpp | 5 +- .../object-store/impl/collection_notifier.cpp | 17 +- .../object-store/impl/collection_notifier.hpp | 7 +- .../object-store/impl/realm_coordinator.cpp | 40 +- .../object-store/impl/realm_coordinator.hpp | 7 +- src/realm/object-store/object.cpp | 3 +- src/realm/object-store/object.hpp | 2 +- src/realm/object-store/object_accessor.hpp | 2 +- src/realm/object-store/object_store.cpp | 1 + src/realm/object-store/results.cpp | 3 +- src/realm/object-store/results.hpp | 2 +- src/realm/object-store/schema.cpp | 50 +- src/realm/object-store/schema.hpp | 41 +- src/realm/object-store/sectioned_results.cpp | 6 +- src/realm/object-store/sectioned_results.hpp | 10 +- src/realm/object-store/shared_realm.cpp | 98 ++- src/realm/object-store/shared_realm.hpp | 12 +- src/realm/object-store/sync/app.cpp | 28 +- src/realm/object-store/sync/app.hpp | 1 - .../object-store/sync/async_open_task.cpp | 2 +- .../object-store/sync/impl/sync_client.hpp | 45 +- src/realm/object-store/sync/sync_manager.cpp | 2 +- src/realm/object-store/sync/sync_session.cpp | 263 +++----- src/realm/object-store/sync/sync_session.hpp | 50 +- src/realm/object-store/sync/sync_user.cpp | 2 +- src/realm/set.cpp | 43 -- src/realm/set.hpp | 3 - src/realm/sync/CMakeLists.txt | 2 - src/realm/sync/changeset.hpp | 5 +- src/realm/sync/client.cpp | 160 +---- src/realm/sync/client.hpp | 7 +- src/realm/sync/client_base.hpp | 45 +- src/realm/sync/network/default_socket.cpp | 303 +-------- src/realm/sync/network/default_socket.hpp | 79 +-- src/realm/sync/network/http.hpp | 20 +- src/realm/sync/network/network.cpp | 171 +++-- src/realm/sync/network/network.hpp | 15 +- src/realm/sync/network/websocket.cpp | 90 ++- src/realm/sync/network/websocket.hpp | 9 +- src/realm/sync/noinst/client_history_impl.cpp | 56 +- src/realm/sync/noinst/client_history_impl.hpp | 15 +- src/realm/sync/noinst/client_impl_base.cpp | 499 ++++++-------- src/realm/sync/noinst/client_impl_base.hpp | 83 ++- src/realm/sync/noinst/client_reset.cpp | 69 +- .../sync/noinst/client_reset_operation.cpp | 60 +- .../sync/noinst/client_reset_operation.hpp | 4 +- src/realm/sync/noinst/server/server.cpp | 24 +- src/realm/sync/socket_provider.hpp | 16 +- .../sync/tools/apply_to_state_command.cpp | 3 +- src/realm/sync/transform.cpp | 197 ++---- src/realm/sync/transform.hpp | 5 + src/realm/transaction.cpp | 26 +- src/realm/transaction.hpp | 4 +- src/realm/util/basic_system_errors.cpp | 10 +- src/realm/util/basic_system_errors.hpp | 1 - src/realm/util/buffer_stream.hpp | 32 - src/realm/util/encrypted_file_mapping.cpp | 3 +- src/realm/util/features.h | 27 +- src/realm/util/file.cpp | 618 +++++++++--------- src/realm/util/file.hpp | 159 +++-- src/realm/util/file_mapper.cpp | 80 +-- src/realm/util/file_mapper.hpp | 21 +- src/realm/util/interprocess_mutex.hpp | 7 +- src/realm/util/logger.cpp | 18 +- src/realm/util/logger.hpp | 158 ++--- src/realm/util/misc_errors.cpp | 10 +- src/realm/util/misc_errors.hpp | 1 - src/realm/util/timestamp_logger.hpp | 2 +- src/realm/utilities.hpp | 2 +- test/CMakeLists.txt | 3 +- test/benchmark-sync/bench_transform.cpp | 3 - test/fuzzy/libfuzzer_entry.cpp | 13 +- test/object-store/audit.cpp | 330 ++++------ test/object-store/c_api/c_api.cpp | 336 +--------- test/object-store/collection_fixtures.hpp | 39 +- test/object-store/dictionary.cpp | 85 +-- test/object-store/list.cpp | 68 -- test/object-store/migrations.cpp | 327 +++++++-- test/object-store/object.cpp | 96 +-- test/object-store/primitive_list.cpp | 9 +- test/object-store/realm.cpp | 424 +++--------- test/object-store/results.cpp | 54 -- test/object-store/set.cpp | 70 -- test/object-store/sync/app.cpp | 329 +--------- test/object-store/sync/client_reset.cpp | 255 +------- test/object-store/sync/flx_sync.cpp | 156 +---- test/object-store/sync/flx_sync_harness.hpp | 9 +- test/object-store/sync/session/session.cpp | 99 --- .../sync/session/session_util.hpp | 2 - test/object-store/sync/sync_test_utils.cpp | 90 +-- test/object-store/sync/sync_test_utils.hpp | 6 - test/object-store/util/baas_admin_api.cpp | 18 +- test/object-store/util/baas_admin_api.hpp | 4 +- test/object-store/util/test_file.cpp | 20 +- test/object-store/util/test_file.hpp | 6 +- test/realm-fuzzer/CMakeLists.txt | 45 -- test/realm-fuzzer/README.md | 72 -- test/realm-fuzzer/afl_runner.cpp | 36 - test/realm-fuzzer/fuzz_configurator.cpp | 108 --- test/realm-fuzzer/fuzz_configurator.hpp | 52 -- test/realm-fuzzer/fuzz_engine.cpp | 182 ------ test/realm-fuzzer/fuzz_engine.hpp | 41 -- test/realm-fuzzer/fuzz_logger.hpp | 48 -- test/realm-fuzzer/fuzz_object.cpp | 547 ---------------- test/realm-fuzzer/fuzz_object.hpp | 71 -- test/realm-fuzzer/libfuzzer_runner.cpp | 32 - test/realm-fuzzer/scripts/start_fuzz_afl.sh | 107 --- test/realm-fuzzer/scripts/start_lib_fuzzer.sh | 47 -- test/realm-fuzzer/util.hpp | 79 --- test/sync_fixtures.hpp | 96 +-- test/test_client_reset.cpp | 44 -- test/test_file.cpp | 11 +- test/test_file_locks.cpp | 8 +- test/test_handshake.cpp | 523 +++++++++++++++ test/test_metrics.cpp | 9 +- test/test_mixed_null_assertions.cpp | 7 - test/test_set.cpp | 49 -- test/test_shared.cpp | 12 +- test/test_sync.cpp | 124 ++-- test/test_sync_history_migration.cpp | 1 - test/test_util_error.cpp | 9 +- test/test_util_file.cpp | 29 +- test/test_util_http.cpp | 10 +- test/test_util_logger.cpp | 108 +-- test/test_util_network.cpp | 64 -- test/test_util_websocket.cpp | 56 +- test/util/CMakeLists.txt | 1 + test/util/test_logger.hpp | 129 ++++ test/util/unit_test.cpp | 43 +- test/util/unit_test.hpp | 35 +- tools/cmake/SpecialtyBuilds.cmake | 2 +- 182 files changed, 3268 insertions(+), 8062 deletions(-) delete mode 100644 .github/advanced-issue-labeler.yml delete mode 100644 .github/auto_assign.yml delete mode 100644 .github/workflows/auto-assign.yml delete mode 100644 .github/workflows/check-changelog.yml delete mode 100644 .github/workflows/issue-labeler.yml delete mode 100644 .github/workflows/no-response.yml delete mode 100644 evergreen/add_admin_roles.js rename src/realm/{sync => object-store}/binding_callback_thread_observer.cpp (92%) rename src/realm/{sync => object-store}/binding_callback_thread_observer.hpp (96%) delete mode 100644 src/realm/object-store/c_api/socket_provider.cpp delete mode 100644 test/realm-fuzzer/CMakeLists.txt delete mode 100644 test/realm-fuzzer/README.md delete mode 100644 test/realm-fuzzer/afl_runner.cpp delete mode 100644 test/realm-fuzzer/fuzz_configurator.cpp delete mode 100644 test/realm-fuzzer/fuzz_configurator.hpp delete mode 100644 test/realm-fuzzer/fuzz_engine.cpp delete mode 100644 test/realm-fuzzer/fuzz_engine.hpp delete mode 100644 test/realm-fuzzer/fuzz_logger.hpp delete mode 100644 test/realm-fuzzer/fuzz_object.cpp delete mode 100644 test/realm-fuzzer/fuzz_object.hpp delete mode 100644 test/realm-fuzzer/libfuzzer_runner.cpp delete mode 100755 test/realm-fuzzer/scripts/start_fuzz_afl.sh delete mode 100755 test/realm-fuzzer/scripts/start_lib_fuzzer.sh delete mode 100644 test/realm-fuzzer/util.hpp create mode 100644 test/test_handshake.cpp create mode 100644 test/util/test_logger.hpp diff --git a/.github/advanced-issue-labeler.yml b/.github/advanced-issue-labeler.yml deleted file mode 100644 index 87a2ae04a61..00000000000 --- a/.github/advanced-issue-labeler.yml +++ /dev/null @@ -1,76 +0,0 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - -# syntax - https://github.com/redhat-plumbers-in-action/advanced-issue-labeler#policy -# Below keys map from the option used in issue form dropdowns to issue labels -# Limitation: -# Currently it's not possible to use strings containing ,␣ in the dropdown menus in the issue forms. - ---- - -policy: - - template: [bug.yml, feature.yml] - section: - - id: [frequency] - label: - - name: 'Frequency:Once' - keys: ['Once'] - - name: 'Frequency:Sometimes' - keys: ['Sometimes'] - - name: 'Frequency:Always' - keys: ['Always'] - - - id: [repro] - label: - - name: 'Repro:Always' - keys: ['Always'] - - name: 'Repro:Sometimes' - keys: ['Sometimes'] - - name: 'Repro:No' - keys: ['No'] - - - id: [sync, flavour, services] - block-list: [] - label: - - name: 'SDK-Use:Local' - keys: ['Local Database only'] - - name: 'SDK-Use:Sync' - keys: ['Atlas Device Sync'] - - name: 'SDK-Use:Services' - keys: ['Atlas App Services: Function or GraphQL or DataAPI etc'] - - name: ['SDK-Use:All'] - keys: ['Both Atlas Device Sync and Atlas App Services'] - - - id: [encryption] - block-list: [] - label: - - name: 'Encryption:On' - keys: ['Yes'] - - name: 'Encryption:Off' - keys: ['No'] - - - id: [app-type] - block-list: [] - label: - - name: 'App-type:Unity' - keys: ['Unity'] - - name: 'App-type:Xamarin' - keys: ['Xamarin'] - - name: 'App-type:WPF' - keys: ['WPF'] - - name: 'App-type:Console' - keys: ['Console or Server'] - - name: 'App-type:Other' - keys: ['Other'] - - - id: [importance] - block-list: [] - label: - - name: 'Importance:Dealbraker' - keys: ['Dealbreaker'] - - name: 'Importance:Major' - keys: ['Would be a major improvement'] - - name: 'Importance:Workaround' - keys: ['I would like to have it but have a workaround'] - - name: 'Importance:Nice' - keys: ['Fairly niche but nice to have anyway'] diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml deleted file mode 100644 index cb3b1df80f1..00000000000 --- a/.github/auto_assign.yml +++ /dev/null @@ -1,3 +0,0 @@ -addAssignees: author -addReviewers: false -runOnDraft: true diff --git a/.github/workflows/Issue-Needs-Attention.yml b/.github/workflows/Issue-Needs-Attention.yml index 842194ac425..b0494c1e398 100644 --- a/.github/workflows/Issue-Needs-Attention.yml +++ b/.github/workflows/Issue-Needs-Attention.yml @@ -1,6 +1,3 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - name: Issue Needs Attention # This workflow is triggered on issue comments. on: diff --git a/.github/workflows/auto-assign.yml b/.github/workflows/auto-assign.yml deleted file mode 100644 index e2861e44908..00000000000 --- a/.github/workflows/auto-assign.yml +++ /dev/null @@ -1,13 +0,0 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - -name: 'Auto Assign' -on: - pull_request: - types: [opened] - -jobs: - add-assignee: - runs-on: ubuntu-latest - steps: - - uses: kentaro-m/auto-assign-action@248761c4feb3917c1b0444e33fad1a50093b9847 diff --git a/.github/workflows/check-changelog.yml b/.github/workflows/check-changelog.yml deleted file mode 100644 index e23ae175127..00000000000 --- a/.github/workflows/check-changelog.yml +++ /dev/null @@ -1,21 +0,0 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - -name: "Check Changelog" -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] - -jobs: - changelog: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@8230315d06ad95c617244d2f265d237a1682d445 - with: - submodules: false - - name: Enforce Changelog - uses: dangoslen/changelog-enforcer@c0b9fd225180a405c5f21f7a74b99e2eccc3e951 - with: - skipLabels: no-changelog - missingUpdateErrorMessage: Please add an entry in CHANGELOG.md or apply the 'no-changelog' label to skip this check. diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml deleted file mode 100644 index 5c9af500c00..00000000000 --- a/.github/workflows/issue-labeler.yml +++ /dev/null @@ -1,35 +0,0 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - -# See configuration in .github/advanced-issue-labeler.yml - -name: Issue labeler (policy) -on: - issues: - types: [ opened ] - -jobs: - label-issues-policy: - runs-on: ubuntu-latest - permissions: - issues: write - - strategy: - matrix: - template: [ bug.yml, feature.yml ] - - steps: - - uses: actions/checkout@v3 - - - name: Parse issue form - uses: stefanbuck/github-issue-parser@c1a559d78bfb8dd05216dab9ffd2b91082ff5324 # v3.0.1 - id: issue-parser - with: - template-path: .github/ISSUE_TEMPLATE/${{ matrix.template }} - - - name: Set labels based on policy - uses: redhat-plumbers-in-action/advanced-issue-labeler@6ee6fddfd744ee26b977e6a0436916f256896971 # v2.0.3 - with: - issue-form: ${{ steps.issue-parser.outputs.jsonString }} - template: ${{ matrix.template }} - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml deleted file mode 100644 index 2505323873d..00000000000 --- a/.github/workflows/no-response.yml +++ /dev/null @@ -1,22 +0,0 @@ -# NOTE: This is a common file that is overwritten by realm/ci-actions sync service -# and should only be modified in that repository. - -name: No Response - -# Both `issue_comment` and `scheduled` event types are required for this Action -# to work properly. -on: - issue_comment: - types: [created] - schedule: - # Schedule at 00:00 every day - - cron: '0 0 * * *' - -jobs: - noResponse: - runs-on: ubuntu-latest - steps: - - uses: lee-dohm/no-response@v0.5.0 - with: - token: ${{ github.token }} - responseRequiredLabel: More-information-needed diff --git a/CHANGELOG.md b/CHANGELOG.md index dbc5743d093..fd547d1085c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,102 +16,6 @@ ---------------------------------------------- -# 13.4.1 Release notes - -### Enhancements -* IntegrationException's which require help from support team mention 'Please contact support' in their message ([#6283](https://github.com/realm/realm-core/pull/6283)) -* Add support for nested and overlapping scopes to the Events API. If multiple scopes are active all events generated will be reported to every active scope ([#6288](https://github.com/realm/realm-core/pull/6288)). - -### Fixed -* App 301/308 redirection support doesn't use new location if metadata mode is set to 'NoMetadata'. ([#6280](https://github.com/realm/realm-core/issues/6280), since v12.9.0) -* Expose ad hoc interface for querying dictionary key changes in the C-API. ([#6228](https://github.com/realm/realm-core/issues/6228), since v10.3.3) -* Client reset with recovery or discard local could fail if there were dangling links in lists that got ressurected while the list was being transferred from the fresh realm ([#6292](https://github.com/realm/realm-core/issues/6292), since v11.5.0) - -### Breaking changes -* None. - -### Compatibility -* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. - ------------ - -### Internals -* The lifecycle of the sync client is now separated from the event loop/socket provider it uses for async I/O/timers. The sync client will wait for all outstanding callbacks/sockets to be closed during destruction. The SyncSocketProvider passed to the sync client must run until after the sync client is destroyed but does not need to be stopped as part of tearing down the sync client. ([PR #6276](https://github.com/realm/realm-core/pull/6276)) -* The default event loop will now keep running until it is explicitly stopped rather than until there are no more timers/IO to process. Previously there was a timer set for very far in the future to force the event loop to keep running. ([PR #6265](https://github.com/realm/realm-core/pull/6265)) -* Disable failing check in Metrics_TransactionTimings test ([PR #6206](https://github.com/realm/realm-core/pull/6206)) - ----------------------------------------------- - -# 13.4.0 Release notes - -### Enhancements -* Improve performance of interprocess mutexes on iOS which don't need to support reader-writer locking. The primary beneficiary of this is beginning and ending read transactions, which is now almost as fast as pre-v13.0.0 ([PR #6258](https://github.com/realm/realm-core/pull/6258)). - -### Fixed -* Sharing Realm files between a Catalyst app and Realm Studio did not properly synchronize access to the Realm file ([PR #6258](https://github.com/realm/realm-core/pull/6258), since v6.21.0). -* Fix websocket redirection after server migration if user is logged in ([#6056](https://github.com/realm/realm-core/issues/6056), since v12.9.0) -* Freezing an immutable Realm would hit an assertion failure ([#6260]https://github.com/realm/realm-core/issues/6260), since v13.3.0). - -### Breaking changes -* None. - -### Compatibility -* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. - ------------ - -### Internals -* Added `REALM_ARCHITECTURE_ARM32` and `REALM_ARCHITECTURE_ARM64` macros to `features.h` for easier platform detection. ([#6256](https://github.com/realm/realm-core/pull/6256)) -* Create the fuzzer framework project in order to run fuzz testing on evergreen ([PR #5940](https://github.com/realm/realm-core/pull/5940)) - ----------------------------------------------- - -# 13.3.0 Release notes - -### Enhancements -* `SyncSession::pause()` and `SyncSession::resume()` allow users to suspend a Realm's sync session until it is explicitly resumed in ([#6183](https://github.com/realm/realm-core/pull/6183)). Previously `SyncSession::log_out()` and `SyncSession::close()` could be resumed under a number of circumstances where `SyncSession::revive_if_needed()` were called (like when freezing a realm) - fixes ([#6085](https://github.com/realm/realm-core/issues/6085)) -* Improve the performance of `Realm::freeze()` by eliminating some redudant work around schema initialization and validation. These optimizations do not apply to Realm::get_frozen_realm() ([PR #6211](https://github.com/realm/realm-core/pull/6211)). -* Include context about what object caused the merge exception in OT ([#6204](https://github.com/realm/realm-core/issues/6204)) -* Add support for `Dictionary::get_keys()`, `Dictionary::contains()`, `Dictionary::find_any()` in the C API. ([#6181](https://github.com/realm/realm-core/issues/6181)) - -### Fixed -* "find first" on Decimal128 field with value NaN does not find objects ([6182](https://github.com/realm/realm-core/issues/6182), since v6.0.0) -* Value in List of Mixed would not be updated if new value is Binary and old value is StringData and the values otherwise matches ([#6201](https://github.com/realm/realm-core/issues/6201), since v6.0.0) -* When client reset with recovery is used and the recovery does not actually result in any new local commits, the sync client may have gotten stuck in a cycle with a `A fatal error occured during client reset: 'A previous 'Recovery' mode reset from did not succeed, giving up on 'Recovery' mode to prevent a cycle'` error message. ([#6195](https://github.com/realm/realm-core/issues/6195), since v11.16.0) -* Fixed diverging history in flexible sync if writes occur during bootstrap to objects that just came into view ([#5804](https://github.com/realm/realm-core/issues/5804), since v11.7.0) -* Fix several data races when opening cached frozen Realms. New frozen Realms were added to the cache and the lock released before they were fully initialized, resulting in races if they were immediately read from the cache on another thread ([PR #6211](https://github.com/realm/realm-core/pull/6211), since v6.0.0). -* Properties and types not present in the requested schema would be missing from the reported schema in several scenarios, such as if the Realm was being opened with a different schema version than the persisted one, and if the new tables or columns were added while the Realm instance did not have an active read transaction ([PR #6211](https://github.com/realm/realm-core/pull/6211), since v13.2.0). -* If a client reset w/recovery or discard local is interrupted while the "fresh" realm is being downloaded, the sync client may crash with a MultpleSyncAgents exception ([#6217](https://github.com/realm/realm-core/issues/6217), since v11.13.0) -* Changesets from the server sent during FLX bootstrapping that are larger than 16MB can cause the sync client to crash with a LogicError (PR [#6218](https://github.com/realm/realm-core/pull/6218), since v12.0.0) -* Online compaction may cause a single commit to take a long time ([#6245](https://github.com/realm/realm-core/pull/6245), since v13.0.0) -* Expose `collection_was_cleared` in the C API ([#6200](https://github.com/realm/realm-core/issues/6200), since v.10.4.0) -* `Set::sort()` used a different sort order from sorting any other collection, including a filtered `Set` ([PR #6238](https://github.com/realm/realm-core/pull/6238), since v13.0.0). -* Fix issue where calling `RealmCoordinator::get_realm(Realm::Config, util::Optional)` would not correctly set `m_schema_version` to `ObjectStore::NotVersioned` if no schema was provided in the config when the realm is first opened ([PR #6236](https://github.com/realm/realm-core/pull/6236), since v10.0.0). - -### Breaking changes -* `SyncSession::log_out()` has been renamed to `SyncSession::force_close()` to reflect what it actually does ([#6183](https://github.com/realm/realm-core/pull/6183)) -* Passing an empty `key_path_array` to `add_notification_callback now` now ignores nested property changes. Pass `std::nullopt` to achieve the old meaning. ([#6122](https://github.com/realm/realm-core/pull/6122)) -* Whether to report the file's complete schema or only the requested schema is now an option on RealmConfig (schema_subset_mode) rather than always being enabled for Additive schema modes. All schema modes which this applies to are now supported ([PR #6211](https://github.com/realm/realm-core/pull/6211)). - -### Compatibility -* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. - ------------ - -### Internals -* Fix failures in Metrics_TransactionTimings core test ([#6164](https://github.com/realm/realm-core/issues/6164)) -* Make log level threshold atomic and shared ([#6009](https://github.com/realm/realm-core/issues/6009)) -* Add c_api error category for resolve errors instead of reporting unknown category, part 2. ([PR #6186](https://github.com/realm/realm-core/pull/6186)) -* Remove `File::is_removed` ([#6222](https://github.com/realm/realm-core/pull/6222)) -* Client reset recovery froze Realms for the callbacks in an invalid way. It is unclear if this resulted in any actual problems. -* Fix default enabled debug output during realm-sync-tests ([#6233](https://github.com/realm/realm-core/issues/6233)) -* Migrate service and event loop into DefaultSyncSocket ([PR #6151](https://github.com/realm/realm-core/pull/6151)) -* Move BindingCallbackThreadObserver from object-store to sync ([PR #6151](https://github.com/realm/realm-core/pull/6151)) -* Update ClientImpl::Connection and DefaultWebSocketImpl to use the new WebSocketObserver callbacks ([PR #6219](https://github.com/realm/realm-core/pull/6219)) -* Switched client reset tests to using private `force_client_reset` server API ([PR #6216](https://github.com/realm/realm-core/pull/6216)) - ----------------------------------------------- - # 13.2.0 Release notes ### Enhancements @@ -121,7 +25,6 @@ * Converting flexible sync realms to bundled and local realms is now supported ([#6076](https://github.com/realm/realm-core/pull/6076)) * Compensating write errors are now surfaced to the SDK/user after the compensating write has been applied in a download message ([#6095](https://github.com/realm/realm-core/pull/6095)). * Normalize sync connection parameters for device information ([#6029](https://github.com/realm/realm-core/issues/6029)) -* Add support for providing custom websocket implementations in the C API ([#5917](https://github.com/realm/realm-core/issues/5917)) ### Fixed * Fix `BadVersion` exceptions which could occur when performing multiple writes on one thread while observing change notifications on another thread ([#6069](https://github.com/realm/realm-core/issues/6069), since v13.0.0). @@ -131,11 +34,11 @@ * Fixed possible segfault in sync client where async callback was using object after being deallocated ([#6053](https://github.com/realm/realm-core/issues/6053), since v11.7.0) * Fixed crash when using client reset with recovery and flexible sync with a single subscription ([#6070](https://github.com/realm/realm-core/issues/6070), since v12.3.0) * Fixed crash with wrong transaction state, during realm migration if realm is frozen due to schema mismatch ([#6144](https://github.com/realm/realm-core/issues/6144), since v13.0.0) - + ### Breaking changes * Core no longer provides any vcpkg infrastructure (the ports submodule and overlay triplets), because it handles dependant libraries internally now. * Allow Realm instances to have a complete view of their schema, if mode is additive. ([PR #5784](https://github.com/realm/realm-core/pull/5784)). -* `realm_sync_immediately_run_file_actions` (c-api) now takes a third argument `bool* did_run` that will be set to the result of `SyncManager::immediately_run_file_actions`. ((#6117)[https://github.com/realm/realm-core/pull/6117]) +* `realm_sync_immediately_run_file_actions` (c-api) now takes a third argument `bool* did_run` that will be set to the result of `SyncManager::immediately_run_file_actions`. ((#6117)[https://github.com/realm/realm-core/pull/6117]) * Device information in sync connection parameters was moved into a new `device_info` structure in App::Config ([PR #6066](https://github.com/realm/realm-core/pull/6066)) * `sdk` is now a required field in the `device_device` structure in App::Config ([PR #6066](https://github.com/realm/realm-core/pull/6066)) @@ -146,7 +49,7 @@ ### Internals * Updates for upcoming Platform Networking feature, including new SyncSocketProvider class. ([PR #6096](https://github.com/realm/realm-core/pull/6096)) -* Update namespaces for files moved to realm/sync/network ([PR #6109](https://github.com/realm/realm-core/pull/6109)) +* Updated namespaces for files moved to realm/sync/network ([PR #6109](https://github.com/realm/realm-core/pull/6109)) * Replace util::network::Trigger with a Sync client custom trigger. ([PR #6121](https://github.com/realm/realm-core/pull/6121)) * Create DefaultSyncSocket class ([PR #6116](https://github.com/realm/realm-core/pull/6116)) * Improve detection of Windows target architecture when downloading prebuild dependencies. ([#6135](https://github.com/realm/realm-core/issues/6135)) @@ -163,10 +66,10 @@ ### Fixed * Fixed `realm_add_realm_refresh_callback` and notify immediately that there is not transaction snapshot to advance to. ([#6075](https://github.com/realm/realm-core/issues/6075), since v12.6.0) * Fix no notification for write transaction that contains only change to backlink property. ([#4994](https://github.com/realm/realm-core/issues/4994), since v11.4.1) - + ### Breaking changes * FLX Subscription API reworked to better match SDK consumption patterns ([#6065](https://github.com/realm/realm-core/pull/6065)). Not all changes are breaking, but listing them all here together. - * `Subscription` is now a plain struct with public fields rather than getter functions + * `Subscription` is now a plain struct with public fields rather than getter functions * `has_name()` and `name()` were merged into a single `optional name` field * `has_name()` and `name()` were merged into a single `optional name` field * `SubscriptionSet` now uses the same types for `iterator` and `const_iterator` since neither was intended to support direct mutability * `SubscriptionSet::get_state_change_notification()` now offers a callback-taking overload @@ -189,7 +92,7 @@ ### Fixed * Not possible to open an encrypted file on a device with a page size bigger than the one on which the file was produced. ([#8030](https://github.com/realm/realm-swift/issues/8030), since v12.11.0) * Fixed `realm_refresh` so it uses an argument value for the refresh result and returns any error conditions as return value. ([#6068](https://github.com/realm/realm-core/pull/6068), since v10.4.0) -* Fixed `realm_compact` to actually do the compaction even if the caller did not provide a `did_compact` argument. ([#6068](https://github.com/realm/realm-core/pull/6068), since v12.7.0) +* Fixed `realm_compact` to actually do the compaction even if the caller did not provide a `did_compact` argument. ([#6068](https://github.com/realm/realm-core/pull/6068), since v12.7.0) ### Breaking changes * ObjectId constructor made explicit, so no more implicit conversions from const char* or array of 12 bytes. It now accepts a StringData. ([#6059](https://github.com/realm/realm-core/pull/6059)) @@ -256,7 +159,7 @@ * Restore fallback to full barrier when F_BARRIERSYNC is not available on Apple platforms. ([PR #6033](https://github.com/realm/realm-core/pull/6033), since v12.12.0) * Validation of Queries constructed by the Fluent QueryBuilder was missing. ([#6034](https://github.com/realm/realm-core/issues/6034), since v12.7.0) * Allow setting values on a Mixed property through the C API ([#5985](https://github.com/realm/realm-core/issues/5985), since v10.5.0) - + ### Breaking changes * `Table::query()` overload taking `vector>` now takes `vector>>` in order to distinguish scalar arguments from single-element lists. ([#5973](https://github.com/realm/realm-core/pull/5973)) * Better error handling for `realm_async_begin_write` and `realm_async_commit`. ([#PR6039](https://github.com/realm/realm-core/pull/6039)) diff --git a/LICENSE b/LICENSE index 66a27ec5ff9..f13a8433793 100644 --- a/LICENSE +++ b/LICENSE @@ -174,4 +174,4 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - + \ No newline at end of file diff --git a/Package.swift b/Package.swift index c0bbf9c3511..10bee1897fe 100644 --- a/Package.swift +++ b/Package.swift @@ -3,7 +3,7 @@ import PackageDescription import Foundation -let versionStr = "13.4.1" +let versionStr = "13.2.0" let versionPieces = versionStr.split(separator: "-") let versionCompontents = versionPieces[0].split(separator: ".") let versionExtra = versionPieces.count > 1 ? versionPieces[1] : "" @@ -93,7 +93,6 @@ let notSyncServerSources: [String] = [ "realm/spec.cpp", "realm/status.cpp", "realm/string_data.cpp", - "realm/sync/binding_callback_thread_observer.cpp", "realm/sync/changeset.cpp", "realm/sync/changeset_encoder.cpp", "realm/sync/changeset_parser.cpp", diff --git a/dependencies.list b/dependencies.list index 4e15430f0f8..d29e318e495 100644 --- a/dependencies.list +++ b/dependencies.list @@ -1,5 +1,5 @@ PACKAGE_NAME=realm-core -VERSION=13.4.1 -OPENSSL_VERSION=3.0.8 +VERSION=13.2.0 +OPENSSL_VERSION=3.0.7 WIN32_ZLIB_VERSION=1.2.13 -MDBREALM_TEST_SERVER_TAG=2023-01-19 +MDBREALM_TEST_SERVER_TAG=2022-10-21 diff --git a/evergreen/add_admin_roles.js b/evergreen/add_admin_roles.js deleted file mode 100644 index c853ff141e8..00000000000 --- a/evergreen/add_admin_roles.js +++ /dev/null @@ -1,13 +0,0 @@ -let user_doc = db.users.findOne({"data.email" : "unique_user@domain.com"}); -if (!user_doc) { - throw "could not find admin user!"; -} - -let update_res = db.users.updateOne({"_id" : user_doc._id}, { - "$addToSet" : - {"roles" : {"$each" : [ {"roleName" : "GLOBAL_STITCH_ADMIN"}, {"roleName" : "GLOBAL_BAAS_FEATURE_ADMIN"} ]}} -}); - -if (update_res.modifiedCount != 1) { - throw "could not update admin user!"; -} diff --git a/evergreen/config.yml b/evergreen/config.yml index e1c1bfcec57..530cf191864 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -32,7 +32,7 @@ functions: script: |- set -o errexit git submodule update --init --recursive - + "compile": - command: shell.exec params: @@ -81,10 +81,6 @@ functions: set_cmake_var realm_vars REALM_ENABLE_ALLOC_SET_ZERO BOOL On fi - if [ -n "${enable_fuzzer|}" ]; then - set_cmake_var realm_vars REALM_LIBFUZZER BOOL On - fi - if [ -z "${disable_sync|}" ]; then set_cmake_var realm_vars REALM_ENABLE_SYNC BOOL On fi @@ -228,35 +224,6 @@ functions: content_type: text/text display_name: install baas output optional: true - - "upload fuzzer results": - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - script: |- - if ls crash-*> /dev/null 2>&1; then - echo "Found crash file" - mv crash-* realm-fuzzer-crash.txt - fi - - - command: s3.put - params: - working_dir: realm-core/build/test/realm-fuzzer - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/build/test/realm-fuzzer/realm-fuzzer-crash.txt' - remote_file: '${project}/${branch_name}/${task_id}/${execution}/realm-fuzzer-crash.txt' - bucket: mciuploads - permissions: public-read - content_type: text/text - display_name: Fuzzer crash report - optional: true - - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - script: |- - rm realm-fuzzer-crash.txt "run hang analyzer": - command: shell.exec @@ -625,7 +592,7 @@ tasks: export DEVELOPER_DIR="${xcode_developer_dir}" fi - ./evergreen/install_baas.sh -w ./baas-work-dir -b 1e1df073f20bbf490c0f57086e890aa482855f61 2>&1 | tee install_baas_output.log + ./evergreen/install_baas.sh -w ./baas-work-dir -b 9398abf62ad6876c38cc675bce47b4f96786a1d6 2>&1 | tee install_baas_output.log fi - command: shell.exec @@ -692,15 +659,6 @@ tasks: echo $out exit 1 -- name: fuzzer - commands: - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - shell: /bin/bash - script: |- - ${cmake_build_type|Debug}/realm-libfuzz -rss_limit_mb=0 -max_total_time=3600 - task_groups: - name: compile_test_and_package max_hosts: 1 @@ -772,19 +730,6 @@ task_groups: tasks: - long-running-core-tests -- name: fuzzer-tests - setup_group_can_fail_task: true - setup_group: - - func: "fetch source" - - func: "fetch binaries" - - func: "compile" - vars: - target_to_build: realm-libfuzz - teardown_task: - - func: "upload fuzzer results" - tasks: - - fuzzer - buildvariants: - name: ubuntu2004 display_name: "Ubuntu 20.04 x86_64 (Clang 11)" @@ -917,26 +862,6 @@ buildvariants: distros: - ubuntu2004-large -- name: ubuntu2004-fuzzer - display_name: "Ubuntu 20.04 x86_64 (Clang 11 Fuzzer)" - run_on: ubuntu2004-large - expansions: - clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" - cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-linux-x86_64.tar.gz" - cmake_bindir: "./cmake_binaries/bin" - fetch_missing_dependencies: On - run_tests_against_baas: On - enable_ubsan: On - c_compiler: "./clang_binaries/bin/clang" - cxx_compiler: "./clang_binaries/bin/clang++" - cmake_build_type: RelWithDebInfo - run_with_encryption: On - enable_fuzzer: On - tasks: - - name: fuzzer-tests - cron: "@daily" - patchable: true - - name: rhel70 display_name: "RHEL 7 x86_64" run_on: rhel70-small diff --git a/evergreen/install_baas.sh b/evergreen/install_baas.sh index 87ca895a43c..278e6ead292 100755 --- a/evergreen/install_baas.sh +++ b/evergreen/install_baas.sh @@ -227,7 +227,7 @@ YARN="$WORK_PATH/yarn/bin/yarn" if [[ ! -x "$YARN" ]]; then echo "Getting yarn" mkdir yarn && cd yarn - $CURL -LsS https://yarnpkg.com/latest.tar.gz | tar -xz --strip-components=1 + $CURL -LsS https://s3.amazonaws.com/stitch-artifacts/yarn/latest.tar.gz | tar -xz --strip-components=1 cd - mkdir "$WORK_PATH/yarn_cache" fi @@ -308,7 +308,7 @@ echo "Starting mongodb" echo "Initializing replica set" retries=0 -until "./mongodb-binaries/bin/$MONGOSH" mongodb://localhost:26000/auth --eval 'try { rs.initiate(); } catch (e) { if (e.codeName != "AlreadyInitialized") { throw e; } }' > /dev/null +until "./mongodb-binaries/bin/$MONGOSH" --port 26000 --eval 'try { rs.initiate(); } catch (e) { if (e.codeName != "AlreadyInitialized") { throw e; } }' > /dev/null do if (( retries++ < 5 )); then sleep 1 @@ -337,17 +337,6 @@ go build -o "$WORK_PATH/baas_server" cmd/server/main.go echo $! > "$WORK_PATH/baas_server.pid" "$BASE_PATH"/wait_for_baas.sh "$WORK_PATH/baas_server.pid" -echo "Adding roles to admin user" -$CURL 'http://localhost:9090/api/admin/v3.0/auth/providers/local-userpass/login' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - --silent \ - --fail \ - --output /dev/null \ - --data-raw '{"username":"unique_user@domain.com","password":"password"}' - -"../mongodb-binaries/bin/$MONGOSH" --quiet mongodb://localhost:26000/auth "$BASE_PATH/add_admin_roles.js" - touch "$WORK_PATH/baas_ready" echo "Baas server ready" diff --git a/src/realm.h b/src/realm.h index 1ee1f77938d..b2c713e137b 100644 --- a/src/realm.h +++ b/src/realm.h @@ -370,12 +370,11 @@ typedef enum realm_property_flags { typedef struct realm_notification_token realm_notification_token_t; typedef struct realm_callback_token realm_callback_token_t; typedef struct realm_refresh_callback_token realm_refresh_callback_token_t; +typedef struct realm_thread_observer_token realm_thread_observer_token_t; typedef struct realm_object_changes realm_object_changes_t; typedef struct realm_collection_changes realm_collection_changes_t; -typedef struct realm_dictionary_changes realm_dictionary_changes_t; typedef void (*realm_on_object_change_func_t)(realm_userdata_t userdata, const realm_object_changes_t*); typedef void (*realm_on_collection_change_func_t)(realm_userdata_t userdata, const realm_collection_changes_t*); -typedef void (*realm_on_dictionary_change_func_t)(realm_userdata_t userdata, const realm_dictionary_changes_t*); typedef void (*realm_on_realm_change_func_t)(realm_userdata_t userdata); typedef void (*realm_on_realm_refresh_func_t)(realm_userdata_t userdata); typedef void (*realm_async_begin_write_func_t)(realm_userdata_t userdata); @@ -397,46 +396,6 @@ typedef bool (*realm_scheduler_is_same_as_func_t)(const realm_userdata_t schedul typedef bool (*realm_scheduler_can_deliver_notifications_func_t)(realm_userdata_t userdata); typedef realm_scheduler_t* (*realm_scheduler_default_factory_func_t)(realm_userdata_t userdata); -/* Sync Socket Provider types */ -typedef struct realm_websocket_endpoint { - const char* address; // Host address - uint16_t port; // Host port number - const char* path; // Includes access token in query. - const char** protocols; // Array of one or more websocket protocols - size_t num_protocols; // Number of protocols in array - bool is_ssl; // true if SSL should be used -} realm_websocket_endpoint_t; - -typedef struct realm_sync_socket realm_sync_socket_t; -typedef struct realm_sync_socket_callback realm_sync_socket_callback_t; -typedef void* realm_sync_socket_timer_t; -typedef void* realm_sync_socket_websocket_t; -typedef struct realm_websocket_observer realm_websocket_observer_t; - -typedef void (*realm_sync_socket_post_func_t)(realm_userdata_t userdata, - realm_sync_socket_callback_t* realm_callback); - -typedef realm_sync_socket_timer_t (*realm_sync_socket_create_timer_func_t)( - realm_userdata_t userdata, uint64_t delay_ms, realm_sync_socket_callback_t* realm_callback); - -typedef void (*realm_sync_socket_timer_canceled_func_t)(realm_userdata_t userdata, - realm_sync_socket_timer_t timer_userdata); - -typedef void (*realm_sync_socket_timer_free_func_t)(realm_userdata_t userdata, - realm_sync_socket_timer_t timer_userdata); - -typedef realm_sync_socket_websocket_t (*realm_sync_socket_connect_func_t)( - realm_userdata_t userdata, realm_websocket_endpoint_t endpoint, - realm_websocket_observer_t* realm_websocket_observer); - -typedef void (*realm_sync_socket_websocket_async_write_func_t)(realm_userdata_t userdata, - realm_sync_socket_websocket_t websocket_userdata, - const char* data, size_t size, - realm_sync_socket_callback_t* realm_callback); - -typedef void (*realm_sync_socket_websocket_free_func_t)(realm_userdata_t userdata, - realm_sync_socket_websocket_t websocket_userdata); - /** * Get the VersionID of the current transaction. * @@ -1843,6 +1802,16 @@ RLM_API bool realm_list_clear(realm_list_t*); */ RLM_API bool realm_list_remove_all(realm_list_t*); +/** + * Replace the contents of a list with values. + * + * This is equivalent to calling `realm_list_clear()`, and then + * `realm_list_insert()` repeatedly. + * + * @return True if no exception occurred. + */ +RLM_API bool realm_list_assign(realm_list_t*, const realm_value_t* values, size_t num_values); + /** * Subscribe to notifications for this object. * @@ -1894,11 +1863,10 @@ RLM_API size_t realm_object_changes_get_modified_properties(const realm_object_c * @param out_num_insertions The number of insertions. May be NULL. * @param out_num_modifications The number of modifications. May be NULL. * @param out_num_moves The number of moved elements. May be NULL. - * @param out_collection_was_cleared a flag to signal if the collection has been cleared. May be NULL */ RLM_API void realm_collection_changes_get_num_changes(const realm_collection_changes_t*, size_t* out_num_deletions, size_t* out_num_insertions, size_t* out_num_modifications, - size_t* out_num_moves, bool* out_collection_was_cleared); + size_t* out_num_moves); /** * Get the number of various types of changes in a collection notification, @@ -1916,6 +1884,7 @@ RLM_API void realm_collection_changes_get_num_ranges(const realm_collection_chan size_t* out_num_deletion_ranges, size_t* out_num_insertion_ranges, size_t* out_num_modification_ranges, size_t* out_num_moves); + typedef struct realm_collection_move { size_t from; size_t to; @@ -1972,35 +1941,6 @@ RLM_API void realm_collection_changes_get_ranges( realm_index_range_t* out_modification_ranges_after, size_t max_modification_ranges_after, realm_collection_move_t* out_moves, size_t max_moves); -/** - * Returns the number of changes occured to the dictionary passed as argument - * - * @param changes valid ptr to the dictionary changes structure - * @param out_deletions_size number of deletions - * @param out_insertion_size number of insertions - * @param out_modification_size number of modifications - */ -RLM_API void realm_dictionary_get_changes(const realm_dictionary_changes_t* changes, size_t* out_deletions_size, - size_t* out_insertion_size, size_t* out_modification_size); - -/** - * Returns the list of keys changed for the dictionary passed as argument. - * The user must assure that there is enough memory to accomodate all the keys - * calling `realm_dictionary_get_changes` before. - * - * @param changes valid ptr to the dictionary changes structure - * @param deletions list of deleted keys - * @param deletions_size size of the list of deleted keys - * @param insertions list of inserted keys - * @param insertions_size size of the list of inserted keys - * @param modifications list of modified keys - * @param modification_size size of the list of modified keys - */ -RLM_API void realm_dictionary_get_changed_keys(const realm_dictionary_changes_t* changes, realm_value_t* deletions, - size_t* deletions_size, realm_value_t* insertions, - size_t* insertions_size, realm_value_t* modifications, - size_t* modification_size); - /** * Get a set instance for the property of an object. * @@ -2136,6 +2076,18 @@ RLM_API bool realm_set_clear(realm_set_t*); */ RLM_API bool realm_set_remove_all(realm_set_t*); +/** + * Replace the contents of a set with values. + * + * The provided values may contain duplicates, in which case the size of the set + * after calling this function will be less than @a num_values. + * + * @param values The list of values to insert. + * @param num_values The number of elements. + * @return True if no exception occurred. + */ +RLM_API bool realm_set_assign(realm_set_t*, const realm_value_t* values, size_t num_values); + /** * Subscribe to notifications for this object. * @@ -2284,39 +2236,25 @@ RLM_API realm_object_t* realm_dictionary_get_linked_object(realm_dictionary_t*, RLM_API bool realm_dictionary_erase(realm_dictionary_t*, realm_value_t key, bool* out_erased); /** - * Return the list of keys stored in the dictionary + * Clear a dictionary. * - * @param out_size number of keys - * @param out_keys the list of keys in the dictionary, the memory has to be released once it is no longer used. * @return True if no exception occurred. */ -RLM_API bool realm_dictionary_get_keys(realm_dictionary_t*, size_t* out_size, realm_results_t** out_keys); - -/** - * Check if the dictionary contains a certain key - * - * @param key to search in the dictionary - * @param found True if the such key exists - * @return True if no exception occured - */ -RLM_API bool realm_dictionary_contains_key(const realm_dictionary_t*, realm_value_t key, bool* found); +RLM_API bool realm_dictionary_clear(realm_dictionary_t*); /** - * Check if the dictionary contains a certain value + * Replace the contents of a dictionary with key/value pairs. * - * @param value to search in the dictionary - * @param index the index of the value in the dictionry if such value exists - * @return True if no exception occured - */ -RLM_API bool realm_dictionary_contains_value(const realm_dictionary_t*, realm_value_t value, size_t* index); - - -/** - * Clear a dictionary. + * The provided keys may contain duplicates, in which case the size of the + * dictionary after calling this function will be less than @a num_pairs. * + * @param keys An array of keys of length @a num_pairs. + * @param values An array of values of length @a num_pairs. + * @param num_pairs The number of key-value pairs to assign. * @return True if no exception occurred. */ -RLM_API bool realm_dictionary_clear(realm_dictionary_t*); +RLM_API bool realm_dictionary_assign(realm_dictionary_t*, size_t num_pairs, const realm_value_t* keys, + const realm_value_t* values); /** * Subscribe to notifications for this object. @@ -2326,7 +2264,7 @@ RLM_API bool realm_dictionary_clear(realm_dictionary_t*); RLM_API realm_notification_token_t* realm_dictionary_add_notification_callback(realm_dictionary_t*, realm_userdata_t userdata, realm_free_userdata_func_t userdata_free, realm_key_path_array_t*, - realm_on_dictionary_change_func_t on_change); + realm_on_collection_change_func_t on_change); /** * Get an dictionary from a thread-safe reference, potentially originating in a @@ -3453,7 +3391,6 @@ typedef enum realm_sync_session_state { RLM_SYNC_SESSION_STATE_DYING, RLM_SYNC_SESSION_STATE_INACTIVE, RLM_SYNC_SESSION_STATE_WAITING_FOR_ACCESS_TOKEN, - RLM_SYNC_SESSION_STATE_PAUSED, } realm_sync_session_state_e; typedef enum realm_sync_connection_state { @@ -3584,15 +3521,6 @@ typedef enum realm_sync_error_action { RLM_SYNC_ERROR_ACTION_CLIENT_RESET_NO_RECOVERY, } realm_sync_error_action_e; -typedef enum realm_sync_error_resolve { - RLM_SYNC_ERROR_RESOLVE_HOST_NOT_FOUND = 1, - RLM_SYNC_ERROR_RESOLVE_HOST_NOT_FOUND_TRY_AGAIN = 2, - RLM_SYNC_ERROR_RESOLVE_NO_DATA = 3, - RLM_SYNC_ERROR_RESOLVE_NO_RECOVERY = 4, - RLM_SYNC_ERROR_RESOLVE_SERVICE_NOT_FOUND = 5, - RLM_SYNC_ERROR_RESOLVE_SOCKET_TYPE_NOT_SUPPORTED = 6, -} realm_sync_error_resolve_e; - typedef struct realm_sync_session realm_sync_session_t; typedef struct realm_async_open_task realm_async_open_task_t; @@ -4094,10 +4022,6 @@ RLM_API void realm_sync_session_handle_error_for_testing(const realm_sync_sessio */ RLM_API void realm_register_user_code_callback_error(realm_userdata_t usercode_error) RLM_API_NOEXCEPT; -#if REALM_ENABLE_SYNC - -typedef struct realm_thread_observer_token realm_thread_observer_token_t; - /** * Register a callback handler for bindings interested in registering callbacks before/after the ObjectStore thread * runs. @@ -4112,8 +4036,6 @@ realm_set_binding_callback_thread_observer(realm_on_object_store_thread_callback realm_on_object_store_error_callback_t on_error, realm_userdata_t, realm_free_userdata_func_t free_userdata); -#endif // REALM_ENABLE_SYNC - typedef struct realm_mongodb_collection realm_mongodb_collection_t; typedef struct realm_mongodb_find_options { @@ -4332,53 +4254,5 @@ RLM_API bool realm_mongo_collection_find_one_and_delete(realm_mongodb_collection realm_userdata_t data, realm_free_userdata_func_t delete_data, realm_mongodb_callback_t callback); -typedef enum status_error_code { - STATUS_OK = 0, - STATUS_UNKNOWN_ERROR = 1, - STATUS_RUNTIME_ERROR = 2, - STATUS_LOGIC_ERROR = 3, - STATUS_BROKEN_PROMISE = 4, - STATUS_OPERATION_ABORTED = 5, - - /// WEBSOCKET ERRORS - // STATUS_WEBSOCKET_OK = 1000 IS NOT USED, JUST USE OK INSTEAD - STATUS_WEBSOCKET_GOING_AWAY = 1001, - STATUS_WEBSOCKET_PROTOCOL_ERROR = 1002, - STATUS_WEBSOCKET_UNSUPPORTED_DATA = 1003, - STATUS_WEBSOCKET_RESERVED = 1004, - STATUS_WEBSOCKET_NO_STATUS_RECEIVED = 1005, - STATUS_WEBSOCKET_ABNORMAL_CLOSURE = 1006, - STATUS_WEBSOCKET_INVALID_PAYLOAD_DATA = 1007, - STATUS_WEBSOCKET_POLICY_VIOLATION = 1008, - STATUS_WEBSOCKET_MESSAGE_TOO_BIG = 1009, - STATUS_WEBSOCKET_INAVALID_EXTENSION = 1010, - STATUS_WEBSOCKET_INTERNAL_SERVER_ERROR = 1011, - STATUS_WEBSOCKET_TLS_HANDSHAKE_FAILED = 1015, // USED BY DEFAULT WEBSOCKET -} status_error_code_e; - -RLM_API realm_sync_socket_t* realm_sync_socket_new( - realm_userdata_t userdata, realm_free_userdata_func_t userdata_free, realm_sync_socket_post_func_t post_func, - realm_sync_socket_create_timer_func_t create_timer_func, - realm_sync_socket_timer_canceled_func_t cancel_timer_func, realm_sync_socket_timer_free_func_t free_timer_func, - realm_sync_socket_connect_func_t websocket_connect_func, - realm_sync_socket_websocket_async_write_func_t websocket_write_func, - realm_sync_socket_websocket_free_func_t websocket_free_func); - -RLM_API void realm_sync_socket_callback_complete(realm_sync_socket_callback_t* realm_callback, - status_error_code_e status, const char* reason); - -RLM_API void realm_sync_socket_websocket_connected(realm_websocket_observer_t* realm_websocket_observer, - const char* protocol); - -RLM_API void realm_sync_socket_websocket_error(realm_websocket_observer_t* realm_websocket_observer); - -RLM_API void realm_sync_socket_websocket_message(realm_websocket_observer_t* realm_websocket_observer, - const char* data, size_t data_size); - -RLM_API void realm_sync_socket_websocket_closed(realm_websocket_observer_t* realm_websocket_observer, bool was_clean, - status_error_code_e status, const char* reason); - -RLM_API void realm_sync_client_config_set_sync_socket(realm_sync_client_config_t*, - realm_sync_socket_t*) RLM_API_NOEXCEPT; #endif // REALM_H diff --git a/src/realm/alloc.hpp b/src/realm/alloc.hpp index 4feb5bc4f02..4b50474bed7 100644 --- a/src/realm/alloc.hpp +++ b/src/realm/alloc.hpp @@ -576,7 +576,7 @@ inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, return translate_less_critical(ref_translation_ptr, ref); } } - realm::util::terminate("Invalid ref translation entry", __FILE__, __LINE__, txl.cookie, 0x1234567890, ref, idx); + realm::util::terminate("Invalid ref translation entry", __FILE__, __LINE__, txl.cookie, 0x1234567890); return nullptr; } diff --git a/src/realm/alloc_slab.cpp b/src/realm/alloc_slab.cpp index 191a1c299f3..cea40856435 100644 --- a/src/realm/alloc_slab.cpp +++ b/src/realm/alloc_slab.cpp @@ -41,7 +41,6 @@ #include #include #include -#include using namespace realm; using namespace realm::util; @@ -797,7 +796,8 @@ ref_type SlabAlloc::attach_file(const std::string& file_path, Config& cfg) if (top_ref) { // Get the expected file size by looking up logical file size stored in top array - constexpr size_t max_top_size = (Group::s_file_size_ndx + 1) * 8 + sizeof(Header); + constexpr size_t file_size_ndx = 2; // This MUST match definition of s_file_size_ndx in Group + constexpr size_t max_top_size = (file_size_ndx + 1) * 8 + sizeof(Header); size_t top_page_base = top_ref & ~(page_size() - 1); size_t top_offset = top_ref - top_page_base; size_t map_size = std::min(max_top_size + top_offset, size - top_page_base); @@ -806,13 +806,13 @@ ref_type SlabAlloc::attach_file(const std::string& file_path, Config& cfg) auto top_header = map_top.get_addr() + top_offset; auto top_data = NodeHeader::get_data_from_header(top_header); auto w = NodeHeader::get_width_from_header(top_header); - auto logical_size = size_t(get_direct(top_data, w, Group::s_file_size_ndx)) >> 1; + auto logical_size = size_t(get_direct(top_data, w, file_size_ndx)) >> 1; expected_size = round_up_to_page_size(logical_size); } } - catch (const DecryptionFailed& e) { + catch (const DecryptionFailed&) { note_reader_end(this); - throw InvalidDatabase(util::format("Realm file decryption failed (%1)", e.message()), path); + throw InvalidDatabase("Realm file decryption failed", path); } catch (const std::exception& e) { note_reader_end(this); diff --git a/src/realm/array.hpp b/src/realm/array.hpp index 4a0204f1365..aade6b5a6c7 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -711,7 +711,7 @@ int64_t Array::get(size_t ndx) const noexcept inline int64_t Array::get(size_t ndx) const noexcept { REALM_ASSERT_DEBUG(is_attached()); - REALM_ASSERT_DEBUG_EX(ndx < m_size, ndx, m_size); + REALM_ASSERT_DEBUG(ndx < m_size); return (this->*m_getter)(ndx); // Two ideas that are not efficient but may be worth looking into again: @@ -744,7 +744,7 @@ inline int64_t Array::back() const noexcept inline ref_type Array::get_as_ref(size_t ndx) const noexcept { REALM_ASSERT_DEBUG(is_attached()); - REALM_ASSERT_DEBUG_EX(m_has_refs, m_ref, ndx, m_size); + REALM_ASSERT_DEBUG(m_has_refs); int64_t v = get(ndx); return to_ref(v); } diff --git a/src/realm/db.cpp b/src/realm/db.cpp index 6cadb157184..59928e1de28 100644 --- a/src/realm/db.cpp +++ b/src/realm/db.cpp @@ -511,29 +511,18 @@ class DB::VersionManager { : m_file(file) , m_mutex(mutex) { - size_t size = 0, required_size = sizeof(SharedInfo); - while (size < required_size) { - // Map the file without the lock held. This could result in the - // mapping being too small and having to remap if the file is grown - // concurrently, but if this is the case we should always see a bigger - // size the next time. - auto new_size = static_cast(m_file.get_size()); - REALM_ASSERT(new_size > size); - size = new_size; - m_reader_map.remap(m_file, File::access_ReadWrite, size, File::map_NoSync); - m_info = m_reader_map.get_addr(); - - std::lock_guard lock(m_mutex); - m_local_max_entry = m_info->readers.capacity(); - required_size = sizeof(SharedInfo) + m_info->readers.compute_required_space(m_local_max_entry); - REALM_ASSERT(required_size >= size); - } + std::lock_guard lock(m_mutex); + size_t size = static_cast(m_file.get_size()); + m_reader_map.map(m_file, File::access_ReadWrite, size, File::map_NoSync); + m_info = m_reader_map.get_addr(); + m_local_max_entry = m_info->readers.capacity(); + REALM_ASSERT(sizeof(SharedInfo) + m_info->readers.compute_required_space(m_local_max_entry) == size); } void cleanup_versions(uint64_t& oldest_live_version, TopRefMap& top_refs, bool& any_new_unreachables) { std::lock_guard lock(m_mutex); - ensure_reader_mapping(); + ensure_full_reader_mapping(); m_info->readers.purge_versions(oldest_live_version, top_refs, any_new_unreachables); } @@ -544,25 +533,9 @@ class DB::VersionManager { VersionID get_version_id_of_latest_snapshot() { - { - // First check the local cache. This is an unlocked read, so it may - // race with adding a new version. If this happens we'll either see - // a stale value (acceptable for a racing write on one thread and - // a read on another), or a new value which is guaranteed to not - // be an active index in the local cache. - std::lock_guard lock(m_local_mutex); - auto index = m_info->readers.newest.load(); - if (index < m_local_readers.size()) { - auto& r = m_local_readers[index]; - if (r.is_active()) { - return {r.version, index}; - } - } - } - std::lock_guard lock(m_mutex); - auto index = m_info->readers.newest.load(); - ensure_reader_mapping(index); + ensure_full_reader_mapping(); + uint_fast32_t index = m_info->readers.newest; return {m_info->readers.get(index).version, index}; } @@ -596,39 +569,34 @@ class DB::VersionManager { if (try_grab_local_read_lock(read_lock, type, version_id)) return read_lock; - { - const bool pick_specific = version_id.version != VersionID().version; - std::lock_guard lock(m_mutex); - auto newest = m_info->readers.newest.load(); - REALM_ASSERT(newest != VersionList::nil); - read_lock.m_reader_idx = pick_specific ? version_id.index : newest; - ensure_reader_mapping((unsigned int)read_lock.m_reader_idx); - bool picked_newest = read_lock.m_reader_idx == (unsigned)newest; - auto& r = m_info->readers.get(read_lock.m_reader_idx); - if (pick_specific && version_id.version != r.version) + const bool pick_specific = version_id.version != VersionID().version; + + std::lock_guard lock(m_mutex); + auto newest = m_info->readers.newest.load(); + REALM_ASSERT(newest != VersionList::nil); + read_lock.m_reader_idx = pick_specific ? version_id.index : newest; + ensure_full_reader_mapping(); + bool picked_newest = read_lock.m_reader_idx == (unsigned)newest; + auto& r = m_info->readers.get(read_lock.m_reader_idx); + if (pick_specific && version_id.version != r.version) + throw BadVersion(); + if (!picked_newest) { + if (type == ReadLockInfo::Frozen && r.count_frozen == 0 && r.count_live == 0) + throw BadVersion(); + if (type != ReadLockInfo::Frozen && r.count_live == 0) throw BadVersion(); - if (!picked_newest) { - if (type == ReadLockInfo::Frozen && r.count_frozen == 0 && r.count_live == 0) - throw BadVersion(); - if (type != ReadLockInfo::Frozen && r.count_live == 0) - throw BadVersion(); - } - populate_read_lock(read_lock, r, type); } + populate_read_lock(read_lock, r, type); - { - std::lock_guard local_lock(m_local_mutex); - grow_local_cache(read_lock.m_reader_idx + 1); - auto& r2 = m_local_readers[read_lock.m_reader_idx]; - if (!r2.is_active()) { - r2.version = read_lock.m_version; - r2.filesize = read_lock.m_file_size; - r2.current_top = read_lock.m_top_ref; - r2.count_full = r2.count_live = r2.count_frozen = 0; - } - REALM_ASSERT(field_for_type(r2, type) == 0); - field_for_type(r2, type) = 1; + std::lock_guard local_lock(m_local_mutex); + grow_local_cache(read_lock.m_reader_idx + 1); + auto& r2 = m_local_readers[read_lock.m_reader_idx]; + if (!r2.is_active()) { + r2 = r; + r2.count_full = r2.count_live = r2.count_frozen = 0; } + REALM_ASSERT(field_for_type(r2, type) == 0); + field_for_type(r2, type) = 1; return read_lock; } @@ -636,7 +604,7 @@ class DB::VersionManager { void add_version(ref_type new_top_ref, size_t new_file_size, uint64_t new_version) { std::lock_guard lock(m_mutex); - ensure_reader_mapping(); + ensure_full_reader_mapping(); if (m_info->readers.try_allocate_entry(new_top_ref, new_file_size, new_version)) { return; } @@ -660,17 +628,15 @@ class DB::VersionManager { } private: - void ensure_reader_mapping(unsigned int required = -1) + void ensure_full_reader_mapping() { using _impl::SimulatedFailure; SimulatedFailure::trigger(SimulatedFailure::shared_group__grow_reader_mapping); // Throws - if (required < m_local_max_entry) - return; - - auto new_max_entry = m_info->readers.capacity(); - if (new_max_entry > m_local_max_entry) { + auto index = m_info->readers.capacity() - 1; + if (index >= m_local_max_entry) { // handle mapping expansion if required + auto new_max_entry = m_info->readers.capacity(); size_t info_size = sizeof(DB::SharedInfo) + m_info->readers.compute_required_space(new_max_entry); m_reader_map.remap(m_file, util::File::access_ReadWrite, info_size); // Throws m_local_max_entry = new_max_entry; @@ -821,7 +787,7 @@ void DB::open(const std::string& path, bool no_create_file, const DBOptions& opt File::CloseGuard fcg(m_file); m_file.set_fifo_path(coordination_dir, "lock.fifo"); - if (m_file.try_rw_lock_exclusive()) { // Throws + if (m_file.try_lock_exclusive()) { // Throws File::UnlockGuard ulg(m_file); // We're alone in the world, and it is Ok to initialize the @@ -853,14 +819,12 @@ void DB::open(const std::string& path, bool no_create_file, const DBOptions& opt // macOS has a bug which can cause a hang waiting to obtain a lock, even // if the lock is already open in shared mode, so we work around it by // busy waiting. This should occur only briefly during session initialization. - while (!m_file.try_rw_lock_shared()) { + while (!m_file.try_lock_shared()) { sched_yield(); } #else - m_file.rw_lock_shared(); // Throws + m_file.lock_shared(); // Throws #endif - File::UnlockGuard ulg(m_file); - // The coordination/management dir is created as a side effect of the lock // operation above if needed for lock emulation. But it may also be needed // for other purposes, so make sure it exists. @@ -1270,7 +1234,6 @@ void DB::open(const std::string& path, bool no_create_file, const DBOptions& opt fug_1.release(); // Do not unmap fcg.release(); // Do not close } - ulg.release(); // Do not release shared lock break; } @@ -1458,7 +1421,13 @@ bool DB::compact(bool bump_version_number, util::Optional output_en tr->close_read_with_lock(); m_alloc.detach(); +#ifdef _WIN32 + // can't rename to existing file on Windows + util::File::copy(tmp_path, m_db_path); + util::File::remove(tmp_path); +#else util::File::move(tmp_path, m_db_path); +#endif SlabAlloc::Config cfg; cfg.session_initiator = true; @@ -1625,7 +1594,7 @@ void DB::close_internal(std::unique_lock lock, bool allow_ope // interleave which is not permitted on Windows. It is permitted on *nix. m_file_map.unmap(); m_version_manager.reset(); - m_file.rw_unlock(); + m_file.unlock(); // info->~SharedInfo(); // DO NOT Call destructor m_file.close(); } @@ -2256,8 +2225,8 @@ void DB::low_level_commit(uint_fast64_t new_version, Transaction& transaction, b if (auto limit = out.get_evacuation_limit()) { // Get a work limit based on the size of the transaction we're about to commit - // Add 4k to ensure progress on small commits - size_t work_limit = m_alloc.get_commit_size() / 2 + out.get_free_list_size() + 0x1000; + // Assume at least 4K on top of that for the top arrays + size_t work_limit = 4 * 1024 + m_alloc.get_commit_size() / 2; transaction.cow_outliers(out.get_evacuation_progress(), limit, work_limit); } @@ -2346,7 +2315,7 @@ bool DB::call_with_lock(const std::string& realm_path, CallbackWithLock&& callba lockfile.open(lockfile_path, File::access_ReadWrite, File::create_Auto, 0); // Throws File::CloseGuard fcg(lockfile); lockfile.set_fifo_path(realm_path + ".management", "lock.fifo"); - if (lockfile.try_rw_lock_exclusive()) { // Throws + if (lockfile.try_lock_exclusive()) { // Throws callback(realm_path); return true; } diff --git a/src/realm/db.hpp b/src/realm/db.hpp index 87366f73f19..5a841d01a7c 100644 --- a/src/realm/db.hpp +++ b/src/realm/db.hpp @@ -440,7 +440,6 @@ class DB : public std::enable_shared_from_this { auto res = std::make_unique(); res->m_top_ref = top_ref; res->m_file_size = file_size; - res->m_version = 1; return res; } void check() const noexcept diff --git a/src/realm/decimal128.cpp b/src/realm/decimal128.cpp index f9b878bca28..bf6f630c370 100644 --- a/src/realm/decimal128.cpp +++ b/src/realm/decimal128.cpp @@ -1527,15 +1527,7 @@ bool Decimal128::operator==(const Decimal128& rhs) const noexcept BID_UINT128 l = to_BID_UINT128(*this); BID_UINT128 r = to_BID_UINT128(rhs); bid128_quiet_equal(&ret, &l, &r, &flags); - if (ret) { - return true; - } - bool lhs_is_nan = is_nan(); - bool rhs_is_nan = rhs.is_nan(); - if (lhs_is_nan && rhs_is_nan) { - return m_value.w[1] == rhs.m_value.w[1] && m_value.w[0] == rhs.m_value.w[0]; - } - return 0; + return ret != 0; } bool Decimal128::operator!=(const Decimal128& rhs) const noexcept diff --git a/src/realm/error_codes.cpp b/src/realm/error_codes.cpp index 8d6cec3d552..142eb4042a8 100644 --- a/src/realm/error_codes.cpp +++ b/src/realm/error_codes.cpp @@ -33,18 +33,6 @@ StringData ErrorCodes::error_string(Error code) return "BrokenPromise"; case ErrorCodes::OperationAborted: return "OperationAborted"; - case ErrorCodes::ReadError: - return "ReadError"; - case ErrorCodes::WriteError: - return "WriteError"; - case ErrorCodes::ResolveFailed: - return "ResolveFailed"; - case ErrorCodes::ConnectionFailed: - return "ConnectionFailed"; - case ErrorCodes::WebSocket_Retry_Error: - return "WebSocket: Retry Error"; - case ErrorCodes::WebSocket_Fatal_Error: - return "WebSocket: Fatal Error"; /// WebSocket error codes case ErrorCodes::WebSocket_GoingAway: @@ -72,20 +60,6 @@ StringData ErrorCodes::error_string(Error code) case ErrorCodes::WebSocket_TLSHandshakeFailed: return "WebSocket: TLS Handshake Failed"; - /// WebSocket Errors - reported by server - case ErrorCodes::WebSocket_Unauthorized: - return "WebSocket: Unauthorized"; - case ErrorCodes::WebSocket_Forbidden: - return "WebSocket: Forbidden"; - case ErrorCodes::WebSocket_MovedPermanently: - return "WebSocket: Moved Permanently"; - case ErrorCodes::WebSocket_Client_Too_Old: - return "WebSocket: Client Too Old"; - case ErrorCodes::WebSocket_Client_Too_New: - return "WebSocket: Client Too New"; - case ErrorCodes::WebSocket_Protocol_Mismatch: - return "WebSocket: Protocol Mismatch"; - case ErrorCodes::UnknownError: [[fallthrough]]; default: diff --git a/src/realm/error_codes.hpp b/src/realm/error_codes.hpp index ec51b45b145..c02b552a04d 100644 --- a/src/realm/error_codes.hpp +++ b/src/realm/error_codes.hpp @@ -36,12 +36,6 @@ class ErrorCodes { LogicError = 3, BrokenPromise = 4, OperationAborted = 5, - ReadError = 7, - WriteError = 8, - ResolveFailed = 9, - ConnectionFailed = 10, - WebSocket_Retry_Error = 11, - WebSocket_Fatal_Error = 12, /// WebSocket Errors // WebSocket_OK = 1000 is not used, just use OK instead @@ -57,14 +51,6 @@ class ErrorCodes { WebSocket_InavalidExtension = 1010, WebSocket_InternalServerError = 1011, WebSocket_TLSHandshakeFailed = 1015, // Used by default WebSocket - - /// WebSocket Errors - reported by server - WebSocket_Unauthorized = 4001, - WebSocket_Forbidden = 4002, - WebSocket_MovedPermanently = 4003, - WebSocket_Client_Too_Old = 4004, - WebSocket_Client_Too_New = 4005, - WebSocket_Protocol_Mismatch = 4006, }; static StringData error_string(Error code); diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 2796d873f5e..75c1acb46af 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -50,15 +50,13 @@ set_target_properties(RealmBrowser PROPERTIES ) target_link_libraries(RealmBrowser Storage) -if(REALM_ENABLE_SYNC) add_executable(Realm2JSON realm2json.cpp ) set_target_properties(Realm2JSON PROPERTIES - OUTPUT_NAME "realm2json" + OUTPUT_NAME "realm2json-10" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) -target_link_libraries(Realm2JSON Storage QueryParser Sync) +target_link_libraries(Realm2JSON Storage QueryParser) list(APPEND ExecTargetsToInstall Realm2JSON) -endif() add_executable(RealmDump EXCLUDE_FROM_ALL realm_dump.c) set_target_properties(RealmDump PROPERTIES diff --git a/src/realm/exec/realm2json.cpp b/src/realm/exec/realm2json.cpp index af5f64fea3f..73c07d14157 100644 --- a/src/realm/exec/realm2json.cpp +++ b/src/realm/exec/realm2json.cpp @@ -1,5 +1,4 @@ #include -#include #include const char* legend = @@ -90,12 +89,15 @@ int main(int argc, char const* argv[]) std::string path = argv[argc - 1]; - auto print = [&](realm::TransactionRef tr) { + try { + // First we try to open in read_only mode. In this way we can also open + // realms with a client history + realm::Group g(path); if (output_schema) { - tr->schema_to_json(std::cout, &renames); + g.schema_to_json(std::cout, &renames); } else if (table_filter.size()) { - realm::TableRef target = tr->get_table(table_filter); + realm::TableRef target = g.get_table(table_filter); abort_if(!target, "table not found: '%s'", table_filter.c_str()); realm::Query q = target->query(query_filter); realm::TableView results = q.find_all(); @@ -104,37 +106,22 @@ int main(int argc, char const* argv[]) results.to_json(std::cout, link_depth, renames, output_mode); } else { - tr->to_json(std::cout, link_depth, &renames, output_mode); + g.to_json(std::cout, link_depth, &renames, output_mode); } - }; + } + catch (const realm::FileFormatUpgradeRequired&) { + // In realm history + // Last chance - this one must succeed + auto hist = realm::make_in_realm_history(); + realm::DBOptions options; + options.allow_file_format_upgrade = true; - auto hist = realm::make_in_realm_history(); - realm::DBOptions options; - // First we try to open in read_only mode. - options.allow_file_format_upgrade = false; - options.is_immutable = true; + auto db = realm::DB::create(*hist, path, options); - for (;;) { - try { - auto db = realm::DB::create(*hist, path, options); - if (options.allow_file_format_upgrade) { - std::cerr << "File upgraded to latest version: " << path << std::endl; - } - print(db->start_read()); - return 0; - } - catch (const realm::FileFormatUpgradeRequired&) { - options.allow_file_format_upgrade = true; - options.is_immutable = false; - } - catch (const realm::IncompatibleHistories&) { - hist = realm::sync::make_client_replication(); - options.allow_file_format_upgrade = false; - options.is_immutable = true; - } - catch (...) { - break; - } + std::cerr << "File upgraded to latest version: " << path << std::endl; + + auto tr = db->start_read(); + tr->to_json(std::cout, link_depth, &renames, output_mode); } return 0; diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp index d732e4e4252..d1cc86308cc 100644 --- a/src/realm/exec/realm_trawler.cpp +++ b/src/realm/exec/realm_trawler.cpp @@ -56,9 +56,7 @@ void consolidate_lists(std::vector& list, std::vector& list2) list.insert(list.end(), list2.begin(), list2.end()); list2.clear(); if (list.size() > 1) { - std::sort(begin(list), end(list), [](T& a, T& b) { - return a.start < b.start; - }); + std::sort(begin(list), end(list), [](T& a, T& b) { return a.start < b.start; }); auto prev = list.begin(); for (auto it = list.begin() + 1; it != list.end(); ++it) { @@ -81,11 +79,7 @@ void consolidate_lists(std::vector& list, std::vector& list2) } // Remove all of the now zero-size chunks from the free list - list.erase(std::remove_if(begin(list), end(list), - [](T& chunk) { - return chunk.length == 0; - }), - end(list)); + list.erase(std::remove_if(begin(list), end(list), [](T& chunk) { return chunk.length == 0; }), end(list)); } } @@ -267,7 +261,6 @@ class Table : public Array { m_column_attributes.init(alloc, spec.get_ref(2)); if (spec.size() > 5) { // Must be a Core-6 file. - m_enum_keys.init(alloc, spec.get_ref(4)); m_column_colkeys.init(alloc, spec.get_ref(5)); } else if (spec.size() > 3) { @@ -338,7 +331,6 @@ class Table : public Array { Array m_column_types; Array m_column_names; Array m_column_attributes; - Array m_enum_keys; Array m_column_subspecs; Array m_column_colkeys; Array m_opposite_table; @@ -513,9 +505,7 @@ std::string human_readable(uint64_t val) uint64_t get_size(const std::vector& list) { uint64_t sz = 0; - std::for_each(list.begin(), list.end(), [&](const Entry& e) { - sz += e.length; - }); + std::for_each(list.begin(), list.end(), [&](const Entry& e) { sz += e.length; }); return sz; } @@ -590,9 +580,6 @@ void Table::print_columns(const Group& group) const type_str += "?"; if (attr & realm::col_attr_Indexed) type_str += " (indexed)"; - if (m_enum_keys.valid() && m_enum_keys.get_val(i)) { - type_str += " (enumerated)"; - } std::string star = (m_pk_col && (m_pk_col == col_key)) ? "*" : ""; std::cout << " " << i << ": " << star << m_column_names.get_string(i) << ": " << type_str << std::endl; } @@ -1044,23 +1031,6 @@ void RealmFile::changes() const } } -unsigned int hex_char_to_bin(char c) -{ - if (c >= '0' && c <= '9') - return c - '0'; - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - if (c >= 'A' && c <= 'F') - return c - 'A' + 10; - throw std::invalid_argument("Illegal key (not a hex digit)"); -} - -unsigned int hex_to_bin(char first, char second) -{ - return (hex_char_to_bin(first) << 4) | hex_char_to_bin(second); -} - - int main(int argc, const char* argv[]) { if (argc > 1) { @@ -1074,23 +1044,12 @@ int main(int argc, const char* argv[]) const char* key_ptr = nullptr; char key[64]; for (int curr_arg = 1; curr_arg < argc; curr_arg++) { - if (strcmp(argv[curr_arg], "--keyfile") == 0) { + if (strcmp(argv[curr_arg], "--key") == 0) { std::ifstream key_file(argv[curr_arg + 1]); key_file.read(key, sizeof(key)); key_ptr = key; curr_arg++; } - else if (strcmp(argv[curr_arg], "--hexkey") == 0) { - curr_arg++; - const char* chars = argv[curr_arg]; - if (strlen(chars) != 128) { - throw std::invalid_argument("Key string must be 128 chars long"); - } - for (int idx = 0; idx < 64; ++idx) { - key[idx] = hex_to_bin(chars[idx * 2], chars[idx * 2 + 1]); - } - key_ptr = key; - } else if (strcmp(argv[curr_arg], "--top") == 0) { char* end; curr_arg++; @@ -1148,10 +1107,7 @@ int main(int argc, const char* argv[]) } } else { - std::cout << "Usage: realm-trawler [-afmsw] [--keyfile file-with-binary-crypt-key] [--hexkey " - "crypt-key-in-hex] [--top " - "top_ref] " - << std::endl; + std::cout << "Usage: realm-trawler [-afmsw] [--key crypt_key] [--top top_ref] " << std::endl; std::cout << " f : free list analysis" << std::endl; std::cout << " m : memory leak check" << std::endl; std::cout << " s : schema dump" << std::endl; diff --git a/src/realm/group.cpp b/src/realm/group.cpp index 90a94c17d63..339dd6ae486 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -211,7 +211,7 @@ void Group::set_size() const noexcept int retval = 0; if (is_attached() && m_table_names.is_attached()) { size_t max_index = m_tables.size(); - REALM_ASSERT_EX(max_index < (1 << 16), max_index); + REALM_ASSERT(max_index < (1 << 16)); for (size_t j = 0; j < max_index; ++j) { RefOrTagged rot = m_tables.get_as_ref_or_tagged(j); if (rot.is_ref() && rot.get_as_ref()) { diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 285c2774dda..db1ed3533d7 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -845,7 +845,6 @@ class Group : public ArrayParent { friend class Transaction; friend class TableKeyIterator; friend class CascadeState; - friend class SlabAlloc; }; class TableKeyIterator { diff --git a/src/realm/group_writer.cpp b/src/realm/group_writer.cpp index c6082464b87..33c85f46042 100644 --- a/src/realm/group_writer.cpp +++ b/src/realm/group_writer.cpp @@ -679,8 +679,10 @@ ref_type GroupWriter::write_group() // bigger databases the space required for free lists will be relatively less. max_free_list_size += 10; - size_t max_free_space_needed = - Array::get_max_byte_size(top.size()) + size_per_free_list_entry() * max_free_list_size; + // If current size is less than 128 MB, the database need not expand above 2 GB + // which means that the positions and sizes can still be in 32 bit. + int size_per_entry = (m_logical_size < 0x8000000 ? 8 : 16) + 8; + size_t max_free_space_needed = Array::get_max_byte_size(top.size()) + size_per_entry * max_free_list_size; ALLOC_DBG_COUT(" Allocating file space for freelists:" << std::endl); // Reserve space for remaining arrays. We ask for some extra bytes beyond the diff --git a/src/realm/group_writer.hpp b/src/realm/group_writer.hpp index 41e8b9b4da8..ec63a31ce6e 100644 --- a/src/realm/group_writer.hpp +++ b/src/realm/group_writer.hpp @@ -116,11 +116,6 @@ class GroupWriter : public _impl::ArrayWriterBase { return m_backoff ? 0 : m_evacuation_limit; } - size_t get_free_list_size() - { - return m_free_positions.size() * size_per_free_list_entry(); - } - std::vector& get_evacuation_progress() { return m_evacuation_progress; @@ -259,13 +254,6 @@ class GroupWriter : public _impl::ArrayWriterBase { /// Debug helper - extends the TopRefMap with list of reachable blocks void map_reachable(); - - size_t size_per_free_list_entry() const - { - // If current size is less than 128 MB, the database need not expand above 2 GB - // which means that the positions and sizes can still be in 32 bit. - return (m_logical_size < 0x8000000 ? 8 : 16) + 8; - } }; diff --git a/src/realm/history.cpp b/src/realm/history.cpp index c3ffd2d1f62..97fbbfabbb9 100644 --- a/src/realm/history.cpp +++ b/src/realm/history.cpp @@ -137,14 +137,13 @@ InRealmHistory::version_type InRealmHistory::add_changeset(BinaryData changeset) void InRealmHistory::get_changesets(version_type begin_version, version_type end_version, BinaryIterator* buffer) const noexcept { - REALM_ASSERT_EX(begin_version <= end_version, begin_version, end_version, m_base_version); - REALM_ASSERT_EX(begin_version >= m_base_version, begin_version, end_version, m_base_version); - REALM_ASSERT_EX(end_version <= m_base_version + m_size, end_version, m_base_version, m_size); + REALM_ASSERT(begin_version <= end_version); + REALM_ASSERT(begin_version >= m_base_version); + REALM_ASSERT(end_version <= m_base_version + m_size); version_type n_version_type = end_version - begin_version; version_type offset_version_type = begin_version - m_base_version; - REALM_ASSERT_EX(!util::int_cast_has_overflow(n_version_type) && - !util::int_cast_has_overflow(offset_version_type), - begin_version, end_version, m_base_version); + REALM_ASSERT(!util::int_cast_has_overflow(n_version_type) && + !util::int_cast_has_overflow(offset_version_type)); size_t n = size_t(n_version_type); size_t offset = size_t(offset_version_type); for (size_t i = 0; i < n; ++i) @@ -161,7 +160,7 @@ void InRealmHistory::set_oldest_bound_version(version_type version) // The new changeset is always added before set_oldest_bound_version() // is called. Therefore, the trimming operation can never leave the // history empty. - REALM_ASSERT_EX(num_entries_to_erase < m_size, num_entries_to_erase, m_size); + REALM_ASSERT(num_entries_to_erase < m_size); for (size_t i = 0; i < num_entries_to_erase; ++i) m_changesets->erase(0); // Throws m_base_version += num_entries_to_erase; diff --git a/src/realm/list.cpp b/src/realm/list.cpp index f1e2f93d546..359d59fe6c1 100644 --- a/src/realm/list.cpp +++ b/src/realm/list.cpp @@ -136,37 +136,31 @@ void Lst::sort(std::vector& indices, bool ascending) const { update_if_needed(); - auto tree = m_tree.get(); - if (ascending) { - do_sort(indices, size(), [tree](size_t i1, size_t i2) { - return unresolved_to_null(tree->get(i1)) < unresolved_to_null(tree->get(i2)); - }); + if constexpr (std::is_same_v) { + if (ascending) { + do_sort(indices, size(), [this](size_t i1, size_t i2) { + return get(i1) < get(i2); + }); + } + else { + do_sort(indices, size(), [this](size_t i1, size_t i2) { + return get(i1) > get(i2); + }); + } } else { - do_sort(indices, size(), [tree](size_t i1, size_t i2) { - return unresolved_to_null(tree->get(i1)) > unresolved_to_null(tree->get(i2)); - }); - } -} - -// std::unique, but leaving the minimum value rather than the first found value -// for runs of duplicates. This makes distinct stable without relying on a -// stable sort, which makes it easier to write tests and avoids surprising results -// where distinct appears to change the order of elements -template -static Iterator min_unique(Iterator first, Iterator last, Predicate pred) -{ - if (first == last) { - return first; - } - - Iterator result = first; - while (++first != last) { - bool equal = pred(*result, *first); - if ((equal && *result > *first) || (!equal && ++result != first)) - *result = *first; + auto tree = m_tree.get(); + if (ascending) { + do_sort(indices, size(), [tree](size_t i1, size_t i2) { + return tree->get(i1) < tree->get(i2); + }); + } + else { + do_sort(indices, size(), [tree](size_t i1, size_t i2) { + return tree->get(i1) > tree->get(i2); + }); + } } - return ++result; } template @@ -174,21 +168,26 @@ void Lst::distinct(std::vector& indices, util::Optional sort_or { indices.clear(); sort(indices, sort_order.value_or(true)); - if (indices.empty()) { - return; - } + auto duplicates = indices.end(); - auto tree = m_tree.get(); - auto duplicates = min_unique(indices.begin(), indices.end(), [tree](size_t i1, size_t i2) noexcept { - return unresolved_to_null(tree->get(i1)) == unresolved_to_null(tree->get(i2)); - }); + if constexpr (std::is_same_v) { + duplicates = std::unique(indices.begin(), indices.end(), [this](size_t i1, size_t i2) noexcept { + return get(i1) == get(i2); + }); + } + else { + auto tree = m_tree.get(); + duplicates = std::unique(indices.begin(), indices.end(), [tree](size_t i1, size_t i2) noexcept { + return tree->get(i1) == tree->get(i2); + }); + } // Erase the duplicates indices.erase(duplicates, indices.end()); if (!sort_order) { // Restore original order - std::sort(indices.begin(), indices.end()); + std::sort(indices.begin(), indices.end(), std::less()); } } diff --git a/src/realm/list.hpp b/src/realm/list.hpp index c93030417f6..92380c79c1a 100644 --- a/src/realm/list.hpp +++ b/src/realm/list.hpp @@ -277,20 +277,6 @@ class Lst final : public CollectionBaseImpl> { } } } - -private: - template - static U unresolved_to_null(U value) noexcept - { - return value; - } - - static Mixed unresolved_to_null(Mixed value) noexcept - { - if (value.is_type(type_TypedLink) && value.is_unresolved_link()) - return Mixed{}; - return value; - } }; // Specialization of Lst: @@ -682,7 +668,13 @@ inline T Lst::get(size_t ndx) const throw std::out_of_range("Index out of range"); } - return unresolved_to_null(m_tree->get(ndx)); + auto value = m_tree->get(ndx); + if constexpr (std::is_same_v) { + // return a null for mixed unresolved link + if (value.is_type(type_TypedLink) && value.is_unresolved_link()) + return Mixed{}; + } + return value; } template @@ -885,17 +877,9 @@ T Lst::set(size_t ndx, T value) if (Replication* repl = this->m_obj.get_replication()) { repl->list_set(*this, ndx, value); } - if constexpr (std::is_same_v) { - if (!(old.is_same_type(value) && old == value)) { - do_set(ndx, value); - bump_content_version(); - } - } - else { - if (old != value) { - do_set(ndx, value); - bump_content_version(); - } + if (old != value) { + do_set(ndx, value); + bump_content_version(); } return old; } diff --git a/src/realm/object-store/CMakeLists.txt b/src/realm/object-store/CMakeLists.txt index 14f3d0c6f5d..7e9af655661 100644 --- a/src/realm/object-store/CMakeLists.txt +++ b/src/realm/object-store/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(c_api) set(SOURCES + binding_callback_thread_observer.cpp collection.cpp collection_notifications.cpp dictionary.cpp @@ -37,6 +38,7 @@ set(SOURCES set(HEADERS audit.hpp audit_serializer.hpp + binding_callback_thread_observer.hpp binding_context.hpp collection.hpp collection_notifications.hpp diff --git a/src/realm/object-store/audit.hpp b/src/realm/object-store/audit.hpp index 94e8ea587b6..6ff9c89363e 100644 --- a/src/realm/object-store/audit.hpp +++ b/src/realm/object-store/audit.hpp @@ -82,18 +82,12 @@ class AuditInterface { virtual void update_metadata(std::vector> new_metadata) = 0; // Begin an audit scope. The given `name` is stored in the activity field - // of each generated event. Returns an id which must be used to either - // commit or cancel the scope. - virtual uint64_t begin_scope(std::string_view name) = 0; - // End the scope with the given id and asynchronously save it to disk. The - // optional completion function is called once it has been committed (or an - // error ocurred while trying to do so). - virtual void end_scope(uint64_t, util::UniqueFunction&& completion = nullptr) = 0; - // Cancel the scope with the given id, discarding all events generated. - virtual void cancel_scope(uint64_t) = 0; - // Check if the scope with the given id is currently active and can be - // committed or cancelled. - virtual bool is_scope_valid(uint64_t) = 0; + // of each generated event. + virtual void begin_scope(std::string_view name) = 0; + // End the current scope and asynchronously save it to disk. The optional + // completion function is called once it has been committed (or an error + // ocurred while trying to do so). + virtual void end_scope(util::UniqueFunction&& completion = nullptr) = 0; // Record a custom audit event. Does not use the scope (and does not need to be inside a scope). virtual void record_event(std::string_view activity, util::Optional event_type, util::Optional data, diff --git a/src/realm/object-store/audit.mm b/src/realm/object-store/audit.mm index fc463cfff32..a6cdc168a74 100644 --- a/src/realm/object-store/audit.mm +++ b/src/realm/object-store/audit.mm @@ -349,10 +349,10 @@ void unexpected_instruction() bool operator()(audit_event::Query& query) { - m_logger.trace("Events: Query on %1 at version %2", query.table, query.version); + m_logger.trace("Audit: Query on %1 at version %2", query.table, query.version); if (m_previous_query && m_previous_query->table == query.table && m_previous_query->version == query.version) { - m_logger.trace("Events: merging query into previous query"); + m_logger.trace("Audit: merging query into previous query"); m_previous_query->objects.insert(m_previous_query->objects.end(), query.objects.begin(), query.objects.end()); return true; @@ -364,15 +364,15 @@ bool operator()(audit_event::Query& query) bool operator()(audit_event::Object const& obj) { - m_logger.trace("Events: Object read on %1 %2 at version %3", obj.table, obj.obj, obj.version); + m_logger.trace("Audit: Object read on %1 %2 at version %3", obj.table, obj.obj, obj.version); if (m_previous_query && m_previous_query->table == obj.table && m_previous_query->version == obj.version) { - m_logger.trace("Events: merging read into previous query"); + m_logger.trace("Audit: merging read into previous query"); m_previous_query->objects.push_back(obj.obj); return true; } if (m_previous_obj && m_previous_obj->table == obj.table && m_previous_obj->obj == obj.obj && m_previous_obj->version == obj.version) { - m_logger.trace("Events: discarding duplicate read"); + m_logger.trace("Audit: discarding duplicate read"); return true; } m_previous_obj = &obj; @@ -415,7 +415,7 @@ bool operator()(audit_event::Query& query) }), query.objects.end()); if (query.objects.empty()) - m_logger.trace("Events: discarding empty query on %1", query.table); + m_logger.trace("Audit: discarding empty query on %1", query.table); return query.objects.empty(); } @@ -423,7 +423,7 @@ bool operator()(audit_event::Object const& obj) { bool exists = object_exists(obj.version, obj.table, obj.obj); if (!exists) - m_logger.trace("Events: discarding read on newly created object %1 %2", obj.table, obj.obj); + m_logger.trace("Audit: discarding read on newly created object %1 %2", obj.table, obj.obj); return !exists; } @@ -731,7 +731,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type if (m_current_realm) { auto size = util::File::get_size_static(m_current_realm->config().path); if (size > g_max_partition_size) { - m_logger->info("Events: Closing Realm at '%1': size %2 > max size %3", m_current_realm->config().path, + m_logger->info("Audit: Closing Realm at '%1': size %2 > max size %3", m_current_realm->config().path, size, g_max_partition_size.load()); auto sync_session = m_current_realm->sync_session(); { @@ -744,7 +744,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type wait_for_upload(sync_session); } else { - sync_session->force_close(); + sync_session->log_out(); m_open_paths.erase(m_current_realm->config().path); } } @@ -752,7 +752,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type m_current_realm = nullptr; } else { - m_logger->detail("Events: Reusing existing Realm at '%1'", m_current_realm->config().path); + m_logger->detail("Audit: Reusing existing Realm at '%1'", m_current_realm->config().path); } } @@ -774,7 +774,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type void AuditRealmPool::wait_for_upload(std::shared_ptr session) { - m_logger->info("Events: Uploading '%1'", session->path()); + m_logger->info("Audit: Uploading '%1'", session->path()); m_upload_sessions.push_back(session); session->wait_for_upload_completion([this, weak_self = weak_from_this(), session](std::error_code ec) { auto self = weak_self.lock(); @@ -790,13 +790,13 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type session->close(); m_open_paths.erase(path); if (ec) { - m_logger->error("Events: Upload on '%1' failed with error '%2'.", path, ec.message()); + m_logger->error("Audit: Upload on '%1' failed with error '%2'.", path, ec.message()); if (m_error_handler) { m_error_handler(SyncError(ec, ec.message(), false)); } } else { - m_logger->info("Events: Upload on '%1' completed.", path); + m_logger->info("Audit: Upload on '%1' completed.", path); util::File::remove(path); } if (!m_upload_sessions.empty()) @@ -818,7 +818,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type void AuditRealmPool::scan_for_realms_to_upload() { util::CheckedLockGuard lock(m_mutex); - m_logger->trace("Events: Scanning for Realms in '%1' to upload", m_path_root); + m_logger->trace("Audit: Scanning for Realms in '%1' to upload", m_path_root); util::DirScanner dir(m_path_root); std::string file_name; while (dir.next(file_name)) { @@ -827,15 +827,15 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type std::string path = m_path_root + file_name; if (m_open_paths.count(path)) { - m_logger->trace("Events: Skipping '%1': file is already open", path); + m_logger->trace("Audit: Skipping '%1': file is already open", path); continue; } - m_logger->trace("Events: Checking file '%1'", path); + m_logger->trace("Audit: Checking file '%1'", path); auto db = DB::create(std::make_unique(false), path); auto tr = db->start_read(); if (tr->get_history()->no_pending_local_changes(tr->get_version())) { - m_logger->info("Events: Realm at '%1' is fully uploaded", path); + m_logger->info("Audit: Realm at '%1' is fully uploaded", path); tr = nullptr; db->close(); util::File::remove(path); @@ -879,7 +879,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type sync_config->error_handler = [error_handler = m_error_handler, weak_self = weak_from_this()](auto, SyncError error) { if (auto self = weak_self.lock()) { - self->m_logger->error("Events: Received sync error: %1 (ec=%2)", error.message, error.error_code.value()); + self->m_logger->error("Audit: Received sync error: %1 (ec=%2)", error.message, error.error_code.value()); } if (error_handler) { error_handler(error); @@ -899,7 +899,7 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type config.schema_version = 0; config.sync_config = sync_config; - m_logger->info("Events: Opening new Realm at '%1'", config.path); + m_logger->info("Audit: Opening new Realm at '%1'", config.path); m_current_realm = Realm::get_shared_realm(std::move(config)); util::CheckedLockGuard lock(m_mutex); m_open_paths.insert(m_current_realm->config().path); @@ -920,10 +920,8 @@ bool write_event(Timestamp timestamp, StringData activity, StringData event_type void update_metadata(std::vector> new_metadata) override REQUIRES(!m_mutex); - uint64_t begin_scope(std::string_view name) override REQUIRES(!m_mutex); - void end_scope(uint64_t, util::UniqueFunction&& completion) override REQUIRES(!m_mutex); - void cancel_scope(uint64_t) override REQUIRES(!m_mutex); - bool is_scope_valid(uint64_t) override REQUIRES(!m_mutex); + void begin_scope(std::string_view name) override REQUIRES(!m_mutex); + void end_scope(util::UniqueFunction&& completion) override REQUIRES(!m_mutex); void record_event(std::string_view activity, util::Optional event_type, util::Optional data, util::UniqueFunction&& completion) override REQUIRES(!m_mutex); @@ -940,7 +938,6 @@ void record_event(std::string_view activity, util::Optional event_t private: struct Scope { - uint64_t id; std::shared_ptr metadata; std::string activity_name; std::vector events; @@ -955,14 +952,12 @@ void record_event(std::string_view activity, util::Optional event_t std::shared_ptr m_logger; util::CheckedMutex m_mutex; - std::vector> m_current_scopes GUARDED_BY(m_mutex); - uint64_t m_scope_counter GUARDED_BY(m_mutex) = 0; + std::shared_ptr m_current_scope GUARDED_BY(m_mutex); dispatch_queue_t m_queue; void pin_version(VersionID) REQUIRES(m_mutex); void trigger_write(std::shared_ptr) REQUIRES(m_mutex); void process_scope(AuditContext::Scope& scope) const; - std::vector>::iterator find_scope(uint64_t id) REQUIRES(m_mutex); friend class AuditEventWriter; }; @@ -1033,7 +1028,7 @@ void validate_metadata(std::vector>& metadat void AuditContext::record_query(VersionID version, TableView const& tv) { util::CheckedLockGuard lock(m_mutex); - if (m_current_scopes.empty()) + if (!m_current_scope) return; if (tv.size() == 0) return; // Query didn't match any objects so there wasn't actually a read @@ -1043,16 +1038,14 @@ void validate_metadata(std::vector>& metadat for (size_t i = 0, count = tv.size(); i < count; ++i) objects.push_back(tv.get_key(i)); - audit_event::Query event{now(), version, tv.get_target_table()->get_key(), std::move(objects)}; - for (auto& scope : m_current_scopes) { - scope->events.push_back(event); - } + m_current_scope->events.push_back( + audit_event::Query{now(), version, tv.get_target_table()->get_key(), std::move(objects)}); } void AuditContext::record_read(VersionID version, const Obj& obj, const Obj& parent, ColKey col) { util::CheckedLockGuard lock(m_mutex); - if (m_current_scopes.empty()) + if (!m_current_scope) return; if (obj.get_table()->is_embedded()) return; @@ -1063,30 +1056,23 @@ void validate_metadata(std::vector>& metadat parent_table_key = parent.get_table()->get_key(); parent_obj_key = parent.get_key(); } - - auto obj_key = obj.get_table()->get_key(); - audit_event::Object event{now(), version, obj_key, obj.get_key(), parent_table_key, parent_obj_key, col}; - for (auto& scope : m_current_scopes) { - scope->events.push_back(event); - } + m_current_scope->events.push_back(audit_event::Object{now(), version, obj.get_table()->get_key(), obj.get_key(), + parent_table_key, parent_obj_key, col}); } void AuditContext::prepare_for_write(VersionID old_version) { util::CheckedLockGuard lock(m_mutex); - if (!m_current_scopes.empty()) + if (m_current_scope) pin_version(old_version); } void AuditContext::record_write(VersionID old_version, VersionID new_version) { util::CheckedLockGuard lock(m_mutex); - if (m_current_scopes.empty()) + if (!m_current_scope) return; - audit_event::Write event{now(), old_version, new_version}; - for (auto& scope : m_current_scopes) { - scope->events.push_back(event); - } + m_current_scope->events.push_back(audit_event::Write{now(), old_version, new_version}); } void AuditContext::record_event(std::string_view activity, util::Optional event_type, @@ -1095,8 +1081,7 @@ void validate_metadata(std::vector>& metadat { util::CheckedLockGuard lock(m_mutex); - // scope id isn't used for this scope, so it's just an arbitrary value - auto scope = std::make_shared(Scope{0, m_metadata, std::string(activity)}); + auto scope = std::make_shared(Scope{m_metadata, std::string(activity)}); scope->events.push_back(audit_event::Custom{now(), std::string(activity), event_type, data}); scope->completion = std::move(completion); trigger_write(std::move(scope)); @@ -1104,74 +1089,37 @@ void validate_metadata(std::vector>& metadat void AuditContext::pin_version(VersionID version) { - TransactionRef pin; - for (auto& scope : m_current_scopes) { - bool has_version = - std::any_of(scope->source_transactions.begin(), scope->source_transactions.end(), [&](auto& tr) { - return tr->get_version() == version.version; - }); - if (!has_version) { - if (!pin) { - pin = m_source_db->start_read(version); - } - scope->source_transactions.push_back(pin); - } - } -} - -uint64_t AuditContext::begin_scope(std::string_view name) -{ - util::CheckedLockGuard lock(m_mutex); - auto id = ++m_scope_counter; - m_logger->trace("Events: Beginning event scope '%1' on '%2' named '%3'", id, m_source_db->get_path(), name); - m_current_scopes.push_back(std::make_shared(Scope{id, m_metadata, std::string(name)})); - return id; -} - -std::vector>::iterator AuditContext::find_scope(uint64_t id) -{ - return std::find_if(m_current_scopes.begin(), m_current_scopes.end(), [&](auto& scope) { - return scope->id == id; - }); -} - -void AuditContext::end_scope(uint64_t id, util::UniqueFunction&& completion) -{ - util::CheckedLockGuard lock(m_mutex); - auto it = find_scope(id); - if (it == m_current_scopes.end()) { - throw std::logic_error(util::format( - "Cannot end event scope: scope '%1' not in progress. Scope may have already been ended?", id)); + for (auto& transaction : m_current_scope->source_transactions) { + if (transaction->get_version() == version.version) + return; } - m_logger->trace("Events: Comitting event scope '%1' on '%2' with %3 events", id, m_source_db->get_path(), - (*it)->events.size()); - (*it)->completion = std::move(completion); - trigger_write(std::move(*it)); - m_current_scopes.erase(it); + m_current_scope->source_transactions.push_back(m_source_db->start_read(version)); } -void AuditContext::cancel_scope(uint64_t id) +void AuditContext::begin_scope(std::string_view name) { util::CheckedLockGuard lock(m_mutex); - auto it = find_scope(id); - if (it == m_current_scopes.end()) { - throw std::logic_error(util::format( - "Cannot end event scope: scope '%1' not in progress. Scope may have already been ended?", id)); - } - m_logger->trace("Events: Cancelling event scope '%1' on '%2' with %3 events", id, m_source_db->get_path(), - (*it)->events.size()); - m_current_scopes.erase(it); + if (m_current_scope) + throw std::logic_error("Cannot begin audit scope: audit already in progress"); + m_logger->trace("Audit: Beginning audit scope on '%1' named '%2'", m_source_db->get_path(), name); + m_current_scope = std::make_shared(Scope{m_metadata, std::string(name)}); } -bool AuditContext::is_scope_valid(uint64_t id) +void AuditContext::end_scope(util::UniqueFunction&& completion) { util::CheckedLockGuard lock(m_mutex); - return find_scope(id) != m_current_scopes.end(); + if (!m_current_scope) + throw std::logic_error("Cannot end audit scope: no audit in progress"); + m_logger->trace("Audit: Comitting audit scope on '%1' with %2 events", m_source_db->get_path(), + m_current_scope->events.size()); + m_current_scope->completion = std::move(completion); + trigger_write(std::move(m_current_scope)); + m_current_scope = nullptr; } void AuditContext::process_scope(AuditContext::Scope& scope) const { - m_logger->info("Events: Processing scope for '%1'", m_source_db->get_path()); + m_logger->info("Audit: Processing scope for '%1'", m_source_db->get_path()); try { // Merge single object reads following a query into that query and discard // duplicate reads on objects. @@ -1226,14 +1174,14 @@ void validate_metadata(std::vector>& metadat else { constexpr bool nullable = true; scope.metadata->metadata_cols.push_back(table->add_column(type_String, key, nullable)); - m_logger->trace("Events: Adding column for metadata field '%1'", key); + m_logger->trace("Audit: Adding column for metadata field '%1'", key); } } } AuditEventWriter writer{*m_source_db, *scope.metadata, scope.activity_name, *table, *m_serializer}; - m_logger->trace("Events: Total event count: %1", scope.events.size()); + m_logger->trace("Audit: Total event count: %1", scope.events.size()); // We write directly to the replication log and don't want // the automatic replication to happen @@ -1244,7 +1192,7 @@ void validate_metadata(std::vector>& metadat if (mpark::visit(writer, scope.events[i])) { // This event didn't fit in the current transaction // so commit and try it again after that. - m_logger->detail("Events: Incrementally comitting transaction after %1 events", i); + m_logger->detail("Audit: Incrementally comitting transaction after %1 events", i); tr.commit_and_continue_writing(); --i; } @@ -1254,15 +1202,15 @@ void validate_metadata(std::vector>& metadat if (scope.completion) scope.completion(nullptr); - m_logger->detail("Events: Scope completed"); + m_logger->detail("Audit: Scope completed"); } catch (std::exception const& e) { - m_logger->error("Events: Error when writing scope: %1", e.what()); + m_logger->error("Audit: Error when writing scope: %1", e.what()); if (scope.completion) scope.completion(std::current_exception()); } catch (...) { - m_logger->error("Events: Unknown error when writing scope"); + m_logger->error("Audit: Unknown error when writing scope"); if (scope.completion) scope.completion(std::current_exception()); } diff --git a/src/realm/sync/binding_callback_thread_observer.cpp b/src/realm/object-store/binding_callback_thread_observer.cpp similarity index 92% rename from src/realm/sync/binding_callback_thread_observer.cpp rename to src/realm/object-store/binding_callback_thread_observer.cpp index a6897d2723e..58ce9119fb7 100644 --- a/src/realm/sync/binding_callback_thread_observer.cpp +++ b/src/realm/object-store/binding_callback_thread_observer.cpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include +#include namespace realm { BindingCallbackThreadObserver* g_binding_callback_thread_observer = nullptr; diff --git a/src/realm/sync/binding_callback_thread_observer.hpp b/src/realm/object-store/binding_callback_thread_observer.hpp similarity index 96% rename from src/realm/sync/binding_callback_thread_observer.hpp rename to src/realm/object-store/binding_callback_thread_observer.hpp index bc2d0da2a13..d93af1b5db0 100644 --- a/src/realm/sync/binding_callback_thread_observer.hpp +++ b/src/realm/object-store/binding_callback_thread_observer.hpp @@ -32,7 +32,7 @@ class BindingCallbackThreadObserver { // This method is called just before the thread is being destroyed virtual void will_destroy_thread() = 0; - // This method is called with any exception thrown by client.run(). + // This method is called with any exception throws by client.run(). virtual void handle_error(std::exception const& e) = 0; }; diff --git a/src/realm/object-store/c_api/CMakeLists.txt b/src/realm/object-store/c_api/CMakeLists.txt index 1980351fedd..5f533399b9e 100644 --- a/src/realm/object-store/c_api/CMakeLists.txt +++ b/src/realm/object-store/c_api/CMakeLists.txt @@ -26,7 +26,6 @@ if(REALM_ENABLE_SYNC) http.cpp logging.cpp sync.cpp - socket_provider.cpp ) endif() diff --git a/src/realm/object-store/c_api/conversion.hpp b/src/realm/object-store/c_api/conversion.hpp index 323b0b8c5f9..ef800d86559 100644 --- a/src/realm/object-store/c_api/conversion.hpp +++ b/src/realm/object-store/c_api/conversion.hpp @@ -458,7 +458,6 @@ static inline realm_version_id_t to_capi(const VersionID& v) } realm_sync_error_code_t to_capi(const std::error_code& error_code, std::string& message); -void sync_error_to_error_code(const realm_sync_error_code_t& sync_error_code, std::error_code* error_code_out); } // namespace realm::c_api diff --git a/src/realm/object-store/c_api/dictionary.cpp b/src/realm/object-store/c_api/dictionary.cpp index 0f383f392e0..4853335ff67 100644 --- a/src/realm/object-store/c_api/dictionary.cpp +++ b/src/realm/object-store/c_api/dictionary.cpp @@ -126,34 +126,6 @@ RLM_API bool realm_dictionary_erase(realm_dictionary_t* dict, realm_value_t key, }); } -RLM_API bool realm_dictionary_get_keys(realm_dictionary_t* dict, size_t* out_size, realm_results_t** out_keys) -{ - return wrap_err([&]() { - auto keys = dict->get_keys(); - *out_size = keys.size(); - *out_keys = new realm_results_t{keys}; - return true; - }); -} - -RLM_API bool realm_dictionary_contains_key(const realm_dictionary_t* dict, realm_value_t key, bool* found) -{ - return wrap_err([&]() { - StringData k{key.string.data, key.string.size}; - *found = dict->contains(k); - return true; - }); -} - -RLM_API bool realm_dictionary_contains_value(const realm_dictionary_t* dict, realm_value_t value, size_t* index) -{ - return wrap_err([&]() { - auto val = from_capi(value); - *index = dict->find_any(val); - return true; - }); -} - RLM_API bool realm_dictionary_clear(realm_dictionary_t* dict) { return wrap_err([&]() { diff --git a/src/realm/object-store/c_api/notifications.cpp b/src/realm/object-store/c_api/notifications.cpp index 55ca16b27db..415d06f6324 100644 --- a/src/realm/object-store/c_api/notifications.cpp +++ b/src/realm/object-store/c_api/notifications.cpp @@ -43,30 +43,10 @@ struct CollectionNotificationsCallback { } }; -struct DictionaryNotificationsCallback { - UserdataPtr m_userdata; - realm_on_dictionary_change_func_t m_on_change = nullptr; - - DictionaryNotificationsCallback() = default; - DictionaryNotificationsCallback(DictionaryNotificationsCallback&& other) - : m_userdata(std::exchange(other.m_userdata, nullptr)) - , m_on_change(std::exchange(other.m_on_change, nullptr)) - { - } - - void operator()(const DictionaryChangeSet& changes) - { - if (m_on_change) { - realm_dictionary_changes_t c{changes}; - m_on_change(m_userdata.get(), &c); - } - } -}; - -std::optional build_key_path_array(realm_key_path_array_t* key_path_array) +KeyPathArray build_key_path_array(realm_key_path_array_t* key_path_array) { + KeyPathArray ret; if (key_path_array) { - KeyPathArray ret; for (size_t i = 0; i < key_path_array->nb_elements; i++) { realm_key_path_t* key_path = key_path_array->paths + i; ret.emplace_back(); @@ -76,9 +56,8 @@ std::optional build_key_path_array(realm_key_path_array_t* key_pat kp.emplace_back(TableKey(path_elem->object), ColKey(path_elem->property)); } } - return ret; } - return std::nullopt; + return ret; } } // namespace @@ -157,13 +136,13 @@ RLM_API realm_notification_token_t* realm_set_add_notification_callback(realm_se RLM_API realm_notification_token_t* realm_dictionary_add_notification_callback(realm_dictionary_t* dict, realm_userdata_t userdata, realm_free_userdata_func_t free, realm_key_path_array_t* key_path_array, - realm_on_dictionary_change_func_t on_change) + realm_on_collection_change_func_t on_change) { return wrap_err([&]() { - DictionaryNotificationsCallback cb; + CollectionNotificationsCallback cb; cb.m_userdata = UserdataPtr{userdata, free}; cb.m_on_change = on_change; - auto token = dict->add_key_based_notification_callback(std::move(cb), build_key_path_array(key_path_array)); + auto token = dict->add_notification_callback(std::move(cb), build_key_path_array(key_path_array)); return new realm_notification_token_t{std::move(token)}; }); } @@ -201,8 +180,7 @@ RLM_API void realm_collection_changes_get_num_ranges(const realm_collection_chan RLM_API void realm_collection_changes_get_num_changes(const realm_collection_changes_t* changes, size_t* out_num_deletions, size_t* out_num_insertions, - size_t* out_num_modifications, size_t* out_num_moves, - bool* out_collection_was_cleared) + size_t* out_num_modifications, size_t* out_num_moves) { // FIXME: This has O(n) performance, which seems ridiculous. @@ -214,8 +192,6 @@ RLM_API void realm_collection_changes_get_num_changes(const realm_collection_cha *out_num_modifications = changes->modifications.count(); if (out_num_moves) *out_num_moves = changes->moves.size(); - if (out_collection_was_cleared) - *out_collection_was_cleared = changes->collection_was_cleared; } static inline void copy_index_ranges(const IndexSet& index_set, realm_index_range_t* out_ranges, size_t max) @@ -258,40 +234,6 @@ RLM_API void realm_collection_changes_get_ranges( } } -RLM_API void realm_dictionary_get_changes(const realm_dictionary_changes_t* changes, size_t* out_deletions_size, - size_t* out_insertion_size, size_t* out_modification_size) -{ - if (out_deletions_size) - *out_deletions_size = changes->deletions.size(); - if (out_insertion_size) - *out_insertion_size = changes->insertions.size(); - if (out_modification_size) - *out_modification_size = changes->modifications.size(); -} - -RLM_API void realm_dictionary_get_changed_keys(const realm_dictionary_changes_t* changes, - realm_value_t* deletion_keys, size_t* deletions_size, - realm_value_t* insertion_keys, size_t* insertions_size, - realm_value_t* modification_keys, size_t* modifications_size) -{ - auto fill = [](const auto& collection, realm_value_t* out, size_t* n) { - if (!out || !n) - return; - if (collection.size() == 0 || *n < collection.size()) { - *n = 0; - return; - } - size_t i = 0; - for (auto val : collection) - out[i++] = to_capi(val); - *n = i; - }; - - fill(changes->deletions, deletion_keys, deletions_size); - fill(changes->insertions, insertion_keys, insertions_size); - fill(changes->modifications, modification_keys, modifications_size); -} - static inline void copy_indices(const IndexSet& index_set, size_t* out_indices, size_t max) { size_t i = 0; diff --git a/src/realm/object-store/c_api/realm.cpp b/src/realm/object-store/c_api/realm.cpp index 755226ba733..b76b676a899 100644 --- a/src/realm/object-store/c_api/realm.cpp +++ b/src/realm/object-store/c_api/realm.cpp @@ -17,12 +17,10 @@ realm_refresh_callback_token::~realm_refresh_callback_token() realm::c_api::CBindingContext::get(*m_realm).realm_pending_refresh_callbacks().remove(m_token); } -#if REALM_ENABLE_SYNC realm_thread_observer_token::~realm_thread_observer_token() { realm::g_binding_callback_thread_observer = nullptr; } -#endif // REALM_ENABLE_SYNC namespace realm::c_api { @@ -314,6 +312,7 @@ RLM_API bool realm_remove_table(realm_t* realm, const char* table_name, bool* ta *table_deleted = true; } return true; + ; }); } @@ -356,8 +355,6 @@ void CBindingContext::did_change(std::vector const&, std::vector< m_realm_changed_callbacks.invoke(); } -#if REALM_ENABLE_SYNC - RLM_API realm_thread_observer_token_t* realm_set_binding_callback_thread_observer(realm_on_object_store_thread_callback_t on_thread_create, @@ -386,6 +383,4 @@ realm_set_binding_callback_thread_observer(realm_on_object_store_thread_callback return new realm_thread_observer_token_t(); } -#endif // REALM_ENABLE_SYNC - } // namespace realm::c_api diff --git a/src/realm/object-store/c_api/realm.hpp b/src/realm/object-store/c_api/realm.hpp index 0225a6faa06..46a9e07d8b4 100644 --- a/src/realm/object-store/c_api/realm.hpp +++ b/src/realm/object-store/c_api/realm.hpp @@ -20,10 +20,7 @@ #include #include - -#if REALM_ENABLE_SYNC -#include -#endif // REALM_ENABLE_SYNC +#include namespace realm::c_api { @@ -67,8 +64,6 @@ class CBindingContext : public BindingContext { CallbackRegistry m_schema_changed_callbacks; }; -#if REALM_ENABLE_SYNC - class CBindingThreadObserver : public realm::BindingCallbackThreadObserver { public: using ThreadCallback = util::UniqueFunction; @@ -112,6 +107,4 @@ class CBindingThreadObserver : public realm::BindingCallbackThreadObserver { ErrorCallback m_error_callback; }; -#endif // REALM_ENABLE_SYNC - } // namespace realm::c_api diff --git a/src/realm/object-store/c_api/schema.cpp b/src/realm/object-store/c_api/schema.cpp index bef91f88f84..e5dba1a3596 100644 --- a/src/realm/object-store/c_api/schema.cpp +++ b/src/realm/object-store/c_api/schema.cpp @@ -5,7 +5,7 @@ namespace realm::c_api { RLM_API realm_schema_t* realm_schema_new(const realm_class_info_t* classes, size_t num_classes, - const realm_property_info_t** class_properties) + const realm_property_info** class_properties) { return wrap_err([&]() { std::vector object_schemas; diff --git a/src/realm/object-store/c_api/socket_provider.cpp b/src/realm/object-store/c_api/socket_provider.cpp deleted file mode 100644 index 48701da9e02..00000000000 --- a/src/realm/object-store/c_api/socket_provider.cpp +++ /dev/null @@ -1,262 +0,0 @@ -#include -#include -#include -#include - -namespace realm::c_api { -namespace { - -struct CAPITimer : sync::SyncSocketProvider::Timer { -public: - CAPITimer(realm_userdata_t userdata, int64_t delay_ms, realm_sync_socket_callback_t* handler, - realm_sync_socket_create_timer_func_t create_timer_func, - realm_sync_socket_timer_canceled_func_t cancel_timer_func, - realm_sync_socket_timer_free_func_t free_timer_func) - : m_handler(handler) - , m_timer_create(create_timer_func) - , m_timer_cancel(cancel_timer_func) - , m_timer_free(free_timer_func) - { - m_timer = m_timer_create(userdata, delay_ms, handler); - } - - /// Cancels the timer and destroys the timer instance. - ~CAPITimer() - { - m_timer_cancel(m_userdata, m_timer); - m_timer_free(m_userdata, m_timer); - realm_release(m_handler); - } - - /// Cancel the timer immediately. - void cancel() override - { - m_timer_cancel(m_userdata, m_timer); - } - -private: - realm_sync_socket_timer_t m_timer = nullptr; - - realm_userdata_t m_userdata = nullptr; - realm_sync_socket_callback_t* m_handler = nullptr; - realm_sync_socket_create_timer_func_t m_timer_create = nullptr; - realm_sync_socket_timer_canceled_func_t m_timer_cancel = nullptr; - realm_sync_socket_timer_free_func_t m_timer_free = nullptr; -}; - -struct CAPIWebSocket : sync::WebSocketInterface { -public: - CAPIWebSocket(realm_userdata_t userdata, realm_sync_socket_connect_func_t websocket_connect_func, - realm_sync_socket_websocket_async_write_func_t websocket_write_func, - realm_sync_socket_websocket_free_func_t websocket_free_func, realm_websocket_observer_t* observer, - sync::WebSocketEndpoint&& endpoint) - : m_observer(observer) - , m_userdata(userdata) - , m_websocket_connect(websocket_connect_func) - , m_websocket_async_write(websocket_write_func) - , m_websocket_free(websocket_free_func) - { - realm_websocket_endpoint_t capi_endpoint; - capi_endpoint.address = endpoint.address.c_str(); - capi_endpoint.port = endpoint.port; - capi_endpoint.path = endpoint.path.c_str(); - - std::vector protocols; - for (size_t i = 0; i < endpoint.protocols.size(); ++i) { - auto& protocol = endpoint.protocols[i]; - protocols.push_back(protocol.c_str()); - } - capi_endpoint.protocols = protocols.data(); - capi_endpoint.num_protocols = protocols.size(); - capi_endpoint.is_ssl = endpoint.is_ssl; - - m_socket = m_websocket_connect(m_userdata, capi_endpoint, observer); - } - - /// Destroys the web socket instance. - ~CAPIWebSocket() - { - m_websocket_free(m_userdata, m_socket); - realm_release(m_observer); - } - - void async_write_binary(util::Span data, sync::SyncSocketProvider::FunctionHandler&& handler) final - { - auto shared_handler = std::make_shared(std::move(handler)); - m_websocket_async_write(m_userdata, m_socket, data.data(), data.size(), - new realm_sync_socket_callback_t(std::move(shared_handler))); - } - -private: - realm_sync_socket_websocket_t m_socket = nullptr; - realm_websocket_observer_t* m_observer = nullptr; - realm_userdata_t m_userdata = nullptr; - - realm_sync_socket_connect_func_t m_websocket_connect = nullptr; - realm_sync_socket_websocket_async_write_func_t m_websocket_async_write = nullptr; - realm_sync_socket_websocket_free_func_t m_websocket_free = nullptr; -}; - -struct CAPIWebSocketObserver : sync::WebSocketObserver { -public: - CAPIWebSocketObserver(std::unique_ptr observer) - : m_observer(std::move(observer)) - { - } - - ~CAPIWebSocketObserver() = default; - - void websocket_connected_handler(const std::string& protocol) final - { - m_observer->websocket_connected_handler(protocol); - } - - void websocket_error_handler() final - { - m_observer->websocket_error_handler(); - } - - bool websocket_binary_message_received(util::Span data) final - { - return m_observer->websocket_binary_message_received(data); - } - - bool websocket_closed_handler(bool was_clean, Status status) final - { - return m_observer->websocket_closed_handler(was_clean, status); - } - -private: - std::unique_ptr m_observer; -}; - -struct CAPISyncSocketProvider : sync::SyncSocketProvider { - realm_userdata_t m_userdata = nullptr; - realm_free_userdata_func_t m_free = nullptr; - realm_sync_socket_post_func_t m_post = nullptr; - realm_sync_socket_create_timer_func_t m_timer_create = nullptr; - realm_sync_socket_timer_canceled_func_t m_timer_cancel = nullptr; - realm_sync_socket_timer_free_func_t m_timer_free = nullptr; - realm_sync_socket_connect_func_t m_websocket_connect = nullptr; - realm_sync_socket_websocket_async_write_func_t m_websocket_async_write = nullptr; - realm_sync_socket_websocket_free_func_t m_websocket_free = nullptr; - - CAPISyncSocketProvider() = default; - CAPISyncSocketProvider(CAPISyncSocketProvider&& other) - : m_userdata(std::exchange(other.m_userdata, nullptr)) - , m_free(std::exchange(other.m_free, nullptr)) - , m_post(std::exchange(other.m_post, nullptr)) - , m_timer_create(std::exchange(other.m_timer_create, nullptr)) - , m_timer_cancel(std::exchange(other.m_timer_cancel, nullptr)) - , m_timer_free(std::exchange(other.m_timer_free, nullptr)) - , m_websocket_connect(std::exchange(other.m_websocket_connect, nullptr)) - , m_websocket_async_write(std::exchange(other.m_websocket_async_write, nullptr)) - , m_websocket_free(std::exchange(other.m_websocket_free, nullptr)) - { - REALM_ASSERT(m_free); - REALM_ASSERT(m_post); - REALM_ASSERT(m_timer_create); - REALM_ASSERT(m_timer_cancel); - REALM_ASSERT(m_timer_free); - REALM_ASSERT(m_websocket_connect); - REALM_ASSERT(m_websocket_async_write); - REALM_ASSERT(m_websocket_free); - } - - ~CAPISyncSocketProvider() - { - m_free(m_userdata); - } - - std::unique_ptr connect(std::unique_ptr observer, - sync::WebSocketEndpoint&& endpoint) final - { - auto capi_observer = std::make_shared(std::move(observer)); - return std::make_unique(m_userdata, m_websocket_connect, m_websocket_async_write, - m_websocket_free, new realm_websocket_observer_t(capi_observer), - std::move(endpoint)); - } - - void post(FunctionHandler&& handler) final - { - auto shared_handler = std::make_shared(std::move(handler)); - m_post(m_userdata, new realm_sync_socket_callback_t(std::move(shared_handler))); - } - - SyncTimer create_timer(std::chrono::milliseconds delay, FunctionHandler&& handler) final - { - auto shared_handler = std::make_shared(std::move(handler)); - return std::make_unique(m_userdata, delay.count(), - new realm_sync_socket_callback_t(std::move(shared_handler)), - m_timer_create, m_timer_cancel, m_timer_free); - } -}; - -} // namespace - -RLM_API realm_sync_socket_t* realm_sync_socket_new( - realm_userdata_t userdata, realm_free_userdata_func_t userdata_free, realm_sync_socket_post_func_t post_func, - realm_sync_socket_create_timer_func_t create_timer_func, - realm_sync_socket_timer_canceled_func_t cancel_timer_func, realm_sync_socket_timer_free_func_t free_timer_func, - realm_sync_socket_connect_func_t websocket_connect_func, - realm_sync_socket_websocket_async_write_func_t websocket_write_func, - realm_sync_socket_websocket_free_func_t websocket_free_func) -{ - return wrap_err([&]() { - auto capi_socket_provider = std::make_shared(); - capi_socket_provider->m_userdata = userdata; - capi_socket_provider->m_free = userdata_free; - capi_socket_provider->m_post = post_func; - capi_socket_provider->m_timer_create = create_timer_func; - capi_socket_provider->m_timer_cancel = cancel_timer_func; - capi_socket_provider->m_timer_free = free_timer_func; - capi_socket_provider->m_websocket_connect = websocket_connect_func; - capi_socket_provider->m_websocket_async_write = websocket_write_func; - capi_socket_provider->m_websocket_free = websocket_free_func; - return new realm_sync_socket_t(std::move(capi_socket_provider)); - }); -} - -RLM_API void realm_sync_socket_callback_complete(realm_sync_socket_callback* realm_callback, - status_error_code_e status, const char* reason) -{ - auto complete_status = status == status_error_code_e::STATUS_OK - ? Status::OK() - : Status{static_cast(status), reason}; - (*(realm_callback->get()))(complete_status); - realm_release(realm_callback); -} - -RLM_API void realm_sync_socket_websocket_connected(realm_websocket_observer_t* realm_websocket_observer, - const char* protocol) -{ - realm_websocket_observer->get()->websocket_connected_handler(protocol); -} - -RLM_API void realm_sync_socket_websocket_error(realm_websocket_observer_t* realm_websocket_observer) -{ - realm_websocket_observer->get()->websocket_error_handler(); -} - -RLM_API void realm_sync_socket_websocket_message(realm_websocket_observer_t* realm_websocket_observer, - const char* data, size_t data_size) -{ - realm_websocket_observer->get()->websocket_binary_message_received(util::Span{data, data_size}); -} - -RLM_API void realm_sync_socket_websocket_closed(realm_websocket_observer_t* realm_websocket_observer, bool was_clean, - status_error_code_e status, const char* reason) -{ - auto closed_status = status == status_error_code_e::STATUS_OK - ? Status::OK() - : Status{static_cast(status), reason}; - realm_websocket_observer->get()->websocket_closed_handler(was_clean, closed_status); -} - -RLM_API void realm_sync_client_config_set_sync_socket(realm_sync_client_config_t* config, - realm_sync_socket_t* sync_socket) RLM_API_NOEXCEPT -{ - config->socket_provider = *sync_socket; -} - -} // namespace realm::c_api diff --git a/src/realm/object-store/c_api/sync.cpp b/src/realm/object-store/c_api/sync.cpp index d4e7fb8ef37..ab9c1b08c5c 100644 --- a/src/realm/object-store/c_api/sync.cpp +++ b/src/realm/object-store/c_api/sync.cpp @@ -72,7 +72,6 @@ static_assert(realm_sync_session_state_e(SyncSession::State::Dying) == RLM_SYNC_ static_assert(realm_sync_session_state_e(SyncSession::State::Inactive) == RLM_SYNC_SESSION_STATE_INACTIVE); static_assert(realm_sync_session_state_e(SyncSession::State::WaitingForAccessToken) == RLM_SYNC_SESSION_STATE_WAITING_FOR_ACCESS_TOKEN); -static_assert(realm_sync_session_state_e(SyncSession::State::Paused) == RLM_SYNC_SESSION_STATE_PAUSED); static_assert(realm_sync_connection_state_e(SyncSession::ConnectionState::Disconnected) == RLM_SYNC_CONNECTION_STATE_DISCONNECTED); @@ -241,23 +240,21 @@ static_assert(realm_flx_sync_subscription_set_state_e(SubscriptionSet::State::Su static_assert(realm_flx_sync_subscription_set_state_e(SubscriptionSet::State::Uncommitted) == RLM_SYNC_SUBSCRIPTION_UNCOMMITTED); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::host_not_found) == - RLM_SYNC_ERROR_RESOLVE_HOST_NOT_FOUND); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::host_not_found_try_again) == - RLM_SYNC_ERROR_RESOLVE_HOST_NOT_FOUND_TRY_AGAIN); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::no_data) == RLM_SYNC_ERROR_RESOLVE_NO_DATA); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::no_recovery) == RLM_SYNC_ERROR_RESOLVE_NO_RECOVERY); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::service_not_found) == - RLM_SYNC_ERROR_RESOLVE_SERVICE_NOT_FOUND); -static_assert(realm_sync_error_resolve_e(network::ResolveErrors::socket_type_not_supported) == - RLM_SYNC_ERROR_RESOLVE_SOCKET_TYPE_NOT_SUPPORTED); - } // namespace realm_sync_error_code_t to_capi(const std::error_code& error_code, std::string& message) { auto ret = realm_sync_error_code_t(); + // HACK: there isn't a good way to get a hold of "our" system category + // so we have to make one of "our" error codes to access it + const std::error_category* realm_basic_system_category; + { + using namespace realm::util::error; + std::error_code dummy = make_error_code(basic_system_errors::invalid_argument); + realm_basic_system_category = &dummy.category(); + } + const std::error_category& category = error_code.category(); if (category == realm::sync::client_error_category()) { ret.category = RLM_SYNC_ERROR_CATEGORY_CLIENT; @@ -270,7 +267,7 @@ realm_sync_error_code_t to_capi(const std::error_code& error_code, std::string& ret.category = RLM_SYNC_ERROR_CATEGORY_CONNECTION; } } - else if (category == std::system_category() || category == realm::util::error::basic_system_error_category()) { + else if (category == std::system_category() || category == *realm_basic_system_category) { ret.category = RLM_SYNC_ERROR_CATEGORY_SYSTEM; } else if (category == realm::sync::network::resolve_error_category()) { @@ -288,26 +285,25 @@ realm_sync_error_code_t to_capi(const std::error_code& error_code, std::string& return ret; } -void sync_error_to_error_code(const realm_sync_error_code_t& sync_error_code, std::error_code* error_code_out) +static std::error_code sync_error_to_error_code(const realm_sync_error_code_t& sync_error_code) { - if (error_code_out) { - const realm_sync_error_category_e category = sync_error_code.category; - if (category == RLM_SYNC_ERROR_CATEGORY_CLIENT) { - error_code_out->assign(sync_error_code.value, realm::sync::client_error_category()); - } - else if (category == RLM_SYNC_ERROR_CATEGORY_SESSION || category == RLM_SYNC_ERROR_CATEGORY_CONNECTION) { - error_code_out->assign(sync_error_code.value, realm::sync::protocol_error_category()); - } - else if (category == RLM_SYNC_ERROR_CATEGORY_SYSTEM) { - error_code_out->assign(sync_error_code.value, std::system_category()); - } - else if (category == RLM_SYNC_ERROR_CATEGORY_RESOLVE) { - error_code_out->assign(sync_error_code.value, realm::sync::network::resolve_error_category()); - } - else if (category == RLM_SYNC_ERROR_CATEGORY_UNKNOWN) { - error_code_out->assign(sync_error_code.value, realm::util::error::basic_system_error_category()); - } + auto error = std::error_code(); + const realm_sync_error_category_e category = sync_error_code.category; + if (category == RLM_SYNC_ERROR_CATEGORY_CLIENT) { + error.assign(sync_error_code.value, realm::sync::client_error_category()); + } + else if (category == RLM_SYNC_ERROR_CATEGORY_SESSION || category == RLM_SYNC_ERROR_CATEGORY_CONNECTION) { + error.assign(sync_error_code.value, realm::sync::protocol_error_category()); + } + else if (category == RLM_SYNC_ERROR_CATEGORY_SYSTEM) { + error.assign(sync_error_code.value, std::system_category()); + } + else if (category == RLM_SYNC_ERROR_CATEGORY_UNKNOWN) { + using namespace realm::util::error; + std::error_code dummy = make_error_code(basic_system_errors::invalid_argument); + error.assign(sync_error_code.value, dummy.category()); } + return error; } static Query add_ordering_to_realm_query(Query realm_query, const DescriptorOrdering& ordering) @@ -913,12 +909,12 @@ RLM_API const char* realm_sync_session_get_file_path(const realm_sync_session_t* RLM_API void realm_sync_session_pause(realm_sync_session_t* session) noexcept { - (*session)->pause(); + (*session)->log_out(); } RLM_API void realm_sync_session_resume(realm_sync_session_t* session) noexcept { - (*session)->resume(); + (*session)->revive_if_needed(); } RLM_API bool realm_sync_immediately_run_file_actions(realm_app_t* realm_app, const char* sync_path, @@ -1003,8 +999,7 @@ RLM_API void realm_sync_session_handle_error_for_testing(const realm_sync_sessio REALM_ASSERT(session); realm_sync_error_code_t sync_error{static_cast(error_category), error_code, error_message}; - std::error_code err; - sync_error_to_error_code(sync_error, &err); + auto err = sync_error_to_error_code(sync_error); SyncSession::OnlyForTesting::handle_error(*session->get(), {err, error_message, is_fatal}); } diff --git a/src/realm/object-store/c_api/types.hpp b/src/realm/object-store/c_api/types.hpp index f1280a89978..b4278394679 100644 --- a/src/realm/object-store/c_api/types.hpp +++ b/src/realm/object-store/c_api/types.hpp @@ -467,18 +467,6 @@ struct realm_collection_changes : realm::c_api::WrapC, realm::CollectionChangeSe } }; -struct realm_dictionary_changes : realm::c_api::WrapC, realm::DictionaryChangeSet { - explicit realm_dictionary_changes(realm::DictionaryChangeSet changes) - : realm::DictionaryChangeSet(std::move(changes)) - { - } - - realm_dictionary_changes* clone() const override - { - return new realm_dictionary_changes{static_cast(*this)}; - } -}; - struct realm_notification_token : realm::c_api::WrapC, realm::NotificationToken { explicit realm_notification_token(realm::NotificationToken token) : realm::NotificationToken(std::move(token)) @@ -486,6 +474,11 @@ struct realm_notification_token : realm::c_api::WrapC, realm::NotificationToken } }; +struct realm_thread_observer_token : realm::c_api::WrapC { + explicit realm_thread_observer_token() = default; + ~realm_thread_observer_token(); +}; + struct realm_callback_token : realm::c_api::WrapC { protected: realm_callback_token(realm_t* realm, uint64_t token) @@ -786,72 +779,6 @@ struct realm_mongodb_collection : realm::c_api::WrapC, realm::app::MongoCollecti } }; -struct realm_sync_socket : realm::c_api::WrapC, std::shared_ptr { - explicit realm_sync_socket(std::shared_ptr ptr) - : std::shared_ptr(std::move(ptr)) - { - } - - realm_sync_socket* clone() const override - { - return new realm_sync_socket{*this}; - } - - bool equals(const WrapC& other) const noexcept final - { - if (auto ptr = dynamic_cast(&other)) { - return get() == ptr->get(); - } - return false; - } -}; - -struct realm_websocket_observer : realm::c_api::WrapC, std::shared_ptr { - explicit realm_websocket_observer(std::shared_ptr ptr) - : std::shared_ptr(std::move(ptr)) - { - } - - realm_websocket_observer* clone() const override - { - return new realm_websocket_observer{*this}; - } - - bool equals(const WrapC& other) const noexcept final - { - if (auto ptr = dynamic_cast(&other)) { - return get() == ptr->get(); - } - return false; - } -}; - -struct realm_sync_socket_callback : realm::c_api::WrapC, - std::shared_ptr { - explicit realm_sync_socket_callback(std::shared_ptr ptr) - : std::shared_ptr(std::move(ptr)) - { - } - - realm_sync_socket_callback* clone() const override - { - return new realm_sync_socket_callback{*this}; - } - - bool equals(const WrapC& other) const noexcept final - { - if (auto ptr = dynamic_cast(&other)) { - return get() == ptr->get(); - } - return false; - } -}; - -struct realm_thread_observer_token : realm::c_api::WrapC { - explicit realm_thread_observer_token() = default; - ~realm_thread_observer_token(); -}; - #endif // REALM_ENABLE_SYNC #endif // REALM_OBJECT_STORE_C_API_TYPES_HPP diff --git a/src/realm/object-store/collection.cpp b/src/realm/object-store/collection.cpp index 6c833186144..fd9dd2a8bc9 100644 --- a/src/realm/object-store/collection.cpp +++ b/src/realm/object-store/collection.cpp @@ -219,7 +219,7 @@ util::Optional Collection::average(ColKey col) const } NotificationToken Collection::add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array) & + KeyPathArray key_path_array) & { verify_attached(); m_realm->verify_notifications_available(); diff --git a/src/realm/object-store/collection.hpp b/src/realm/object-store/collection.hpp index 5149ff06e3f..fe970f00e39 100644 --- a/src/realm/object-store/collection.hpp +++ b/src/realm/object-store/collection.hpp @@ -122,7 +122,7 @@ class Collection { * @return A `NotificationToken` that is used to identify this callback. */ NotificationToken add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array = std::nullopt) &; + KeyPathArray key_path_array = {}) &; // The object being added to the collection is already a managed embedded object struct InvalidEmbeddedOperationException : public std::logic_error { diff --git a/src/realm/object-store/dictionary.cpp b/src/realm/object-store/dictionary.cpp index 9a54ae672b1..b70c39be180 100644 --- a/src/realm/object-store/dictionary.cpp +++ b/src/realm/object-store/dictionary.cpp @@ -250,7 +250,7 @@ size_t Dictionary::find_any(Mixed value) const return dict().find_any(value); } -bool Dictionary::contains(StringData key) const +bool Dictionary::contains(StringData key) { return dict().contains(key); } @@ -335,10 +335,9 @@ class NotificationHandler { Dictionary::CBFunc m_cb; }; -NotificationToken Dictionary::add_key_based_notification_callback(CBFunc cb, - std::optional key_path_array) & +NotificationToken Dictionary::add_key_based_notification_callback(CBFunc cb, KeyPathArray key_path_array) & { - return add_notification_callback(NotificationHandler(dict(), std::move(cb)), std::move(key_path_array)); + return add_notification_callback(NotificationHandler(dict(), std::move(cb)), key_path_array); } Dictionary Dictionary::freeze(const std::shared_ptr& frozen_realm) const diff --git a/src/realm/object-store/dictionary.hpp b/src/realm/object-store/dictionary.hpp index b3ac21e542c..be4004b7e12 100644 --- a/src/realm/object-store/dictionary.hpp +++ b/src/realm/object-store/dictionary.hpp @@ -101,7 +101,7 @@ class Dictionary : public object_store::Collection { util::Optional try_get_any(StringData key) const; std::pair get_pair(size_t ndx) const; size_t find_any(Mixed value) const final; - bool contains(StringData key) const; + bool contains(StringData key); template void insert(Context&, StringData key, T&& value, CreatePolicy = CreatePolicy::SetLink); @@ -118,8 +118,7 @@ class Dictionary : public object_store::Collection { Results get_values() const; using CBFunc = util::UniqueFunction; - NotificationToken - add_key_based_notification_callback(CBFunc cb, std::optional key_path_array = std::nullopt) &; + NotificationToken add_key_based_notification_callback(CBFunc cb, KeyPathArray key_path_array = {}) &; Iterator begin() const; Iterator end() const; diff --git a/src/realm/object-store/impl/collection_notifier.cpp b/src/realm/object-store/impl/collection_notifier.cpp index cf4071eb67b..b812aefa034 100644 --- a/src/realm/object-store/impl/collection_notifier.cpp +++ b/src/realm/object-store/impl/collection_notifier.cpp @@ -103,14 +103,14 @@ void CollectionNotifier::recalculate_key_path_array() m_any_callbacks_filtered = false; m_key_path_array.clear(); for (const auto& callback : m_callbacks) { - if (!callback.key_path_array) { + if (callback.key_path_array.empty()) { m_all_callbacks_filtered = false; } else { m_any_callbacks_filtered = true; - for (const auto& key_path : *callback.key_path_array) { - m_key_path_array.push_back(key_path); - } + } + for (const auto& key_path : callback.key_path_array) { + m_key_path_array.push_back(key_path); } } } @@ -151,12 +151,11 @@ void CollectionNotifier::release_data() noexcept static bool all_have_filters(std::vector const& callbacks) noexcept { return std::all_of(callbacks.begin(), callbacks.end(), [](auto& cb) { - return !cb.key_path_array; + return !cb.key_path_array.empty(); }); } -uint64_t CollectionNotifier::add_callback(CollectionChangeCallback callback, - std::optional key_path_array) +uint64_t CollectionNotifier::add_callback(CollectionChangeCallback callback, KeyPathArray key_path_array) { m_realm->verify_thread(); @@ -164,7 +163,7 @@ uint64_t CollectionNotifier::add_callback(CollectionChangeCallback callback, // If we're adding a callback with a keypath filter or if previously all // callbacks had filters but this one doesn't we will need to recalculate // the related tables on the background thread. - if (!key_path_array || all_have_filters(m_callbacks)) { + if (!key_path_array.empty() || all_have_filters(m_callbacks)) { m_did_modify_callbacks = true; } @@ -203,7 +202,7 @@ void CollectionNotifier::remove_callback(uint64_t token) // If we're removing a callback with a keypath filter or the last callback // without a keypath filter we will need to recalcuate the related tables // on next run. - if (!old.key_path_array || all_have_filters(m_callbacks)) { + if (!old.key_path_array.empty() || all_have_filters(m_callbacks)) { m_did_modify_callbacks = true; } diff --git a/src/realm/object-store/impl/collection_notifier.hpp b/src/realm/object-store/impl/collection_notifier.hpp index 97ac005e80e..445eecfe595 100644 --- a/src/realm/object-store/impl/collection_notifier.hpp +++ b/src/realm/object-store/impl/collection_notifier.hpp @@ -51,11 +51,9 @@ struct NotificationCallback { // target thread. CollectionChangeBuilder changes_to_deliver; // The filter that this `NotificationCallback` is restricted to. - // if std::nullopt, then no restriction is enforced. - // if empty, then modifications to objects within the collection won't fire notifications. // If not empty, modifications of elements not part of the `key_path_array` // will not invoke a notification. - std::optional key_path_array = std::nullopt; + KeyPathArray key_path_array = {}; // A unique-per-notifier identifier used to unregister the callback. uint64_t token = 0; // We normally want to skip calling the callback if there's no changes, @@ -96,8 +94,7 @@ class CollectionNotifier { * * @return A token which can be passed to `remove_callback()`. */ - uint64_t add_callback(CollectionChangeCallback callback, std::optional key_path_array) - REQUIRES(!m_callback_mutex); + uint64_t add_callback(CollectionChangeCallback callback, KeyPathArray key_path_array) REQUIRES(!m_callback_mutex); /** * Remove a previously added token. diff --git a/src/realm/object-store/impl/realm_coordinator.cpp b/src/realm/object-store/impl/realm_coordinator.cpp index 057c57d87bb..c3708813ccd 100644 --- a/src/realm/object-store/impl/realm_coordinator.cpp +++ b/src/realm/object-store/impl/realm_coordinator.cpp @@ -238,7 +238,6 @@ std::shared_ptr RealmCoordinator::do_get_cached_realm(Realm::Config const std::shared_ptr RealmCoordinator::get_realm(Realm::Config config, util::Optional version) { - REALM_ASSERT(!version || *version != VersionID()); if (!config.scheduler) config.scheduler = version ? util::Scheduler::make_frozen(*version) : util::Scheduler::make_default(); // realm must be declared before lock so that the mutex is released before @@ -248,13 +247,12 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config, util::O util::CheckedUniqueLock lock(m_realm_mutex); set_config(config); if ((realm = do_get_cached_realm(config))) { - REALM_ASSERT(!version || realm->read_transaction_version() == *version); + if (version) { + REALM_ASSERT(realm->read_transaction_version() == *version); + } return realm; } do_get_realm(std::move(config), realm, version, lock); - if (version) { - realm->read_group(); - } return realm; } @@ -271,34 +269,15 @@ std::shared_ptr RealmCoordinator::get_realm(std::shared_ptr RealmCoordinator::freeze_realm(const Realm& source_realm) -{ - std::shared_ptr realm; - util::CheckedUniqueLock lock(m_realm_mutex); - - auto version = source_realm.read_transaction_version(); - auto scheduler = util::Scheduler::make_frozen(version); - if ((realm = do_get_cached_realm(source_realm.config(), scheduler))) { - return realm; - } - - auto config = source_realm.config(); - config.scheduler = scheduler; - realm = Realm::make_shared_realm(std::move(config), version, shared_from_this()); - Realm::Internal::copy_schema(*realm, source_realm); - m_weak_realm_notifiers.emplace_back(realm, config.cache); - return realm; -} - ThreadSafeReference RealmCoordinator::get_unbound_realm() { std::shared_ptr realm; util::CheckedUniqueLock lock(m_realm_mutex); - do_get_realm(RealmConfig(m_config), realm, none, lock); + do_get_realm(m_config, realm, none, lock); return ThreadSafeReference(realm); } -void RealmCoordinator::do_get_realm(RealmConfig&& config, std::shared_ptr& realm, +void RealmCoordinator::do_get_realm(Realm::Config config, std::shared_ptr& realm, util::Optional version, util::CheckedUniqueLock& realm_lock) { open_db(); @@ -326,15 +305,6 @@ void RealmCoordinator::do_get_realm(RealmConfig&& config, std::shared_ptr REALM_TERMINATE("Cannot use Audit interface if Realm Core is built without Sync"); #endif - // Cached frozen Realms need to initialize their schema before releasing - // the lock as otherwise they could be read from the cache on another thread - // before the schema initialization happens. They'll never perform a write - // transaction, so unlike with live Realms this is safe to do. - if (config.cache && version && schema) { - realm->update_schema(std::move(*schema)); - schema.reset(); - } - realm_lock.unlock_unchecked(); if (schema) { realm->update_schema(std::move(*schema), config.schema_version, std::move(migration_function), diff --git a/src/realm/object-store/impl/realm_coordinator.hpp b/src/realm/object-store/impl/realm_coordinator.hpp index 1ec0169ebe7..dbc5eed947c 100644 --- a/src/realm/object-store/impl/realm_coordinator.hpp +++ b/src/realm/object-store/impl/realm_coordinator.hpp @@ -60,11 +60,6 @@ class RealmCoordinator : public std::enable_shared_from_this { REQUIRES(!m_realm_mutex, !m_schema_cache_mutex); std::shared_ptr get_realm(std::shared_ptr = nullptr) REQUIRES(!m_realm_mutex, !m_schema_cache_mutex); - - // Return a frozen copy of the source Realm. May return a cached instance - // if the source Realm has caching enabled. - std::shared_ptr freeze_realm(const Realm& source_realm) REQUIRES(!m_realm_mutex); - #if REALM_ENABLE_SYNC // Get a thread-local shared Realm with the given configuration // If the Realm is not already present, it will be fully downloaded before being returned. @@ -267,7 +262,7 @@ class RealmCoordinator : public std::enable_shared_from_this { std::shared_ptr do_get_cached_realm(Realm::Config const& config, std::shared_ptr scheduler = nullptr) REQUIRES(m_realm_mutex); - void do_get_realm(Realm::Config&& config, std::shared_ptr& realm, util::Optional version, + void do_get_realm(Realm::Config config, std::shared_ptr& realm, util::Optional version, util::CheckedUniqueLock& realm_lock) REQUIRES(m_realm_mutex); void run_async_notifiers() REQUIRES(!m_notifier_mutex, m_running_notifiers_mutex); void clean_up_dead_notifiers() REQUIRES(m_notifier_mutex); diff --git a/src/realm/object-store/object.cpp b/src/realm/object-store/object.cpp index fb127ecf0f8..2cbb8145e2a 100644 --- a/src/realm/object-store/object.cpp +++ b/src/realm/object-store/object.cpp @@ -155,8 +155,7 @@ Object::Object(Object&&) = default; Object& Object::operator=(Object const&) = default; Object& Object::operator=(Object&&) = default; -NotificationToken Object::add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array) & +NotificationToken Object::add_notification_callback(CollectionChangeCallback callback, KeyPathArray key_path_array) & { verify_attached(); m_realm->verify_notifications_available(); diff --git a/src/realm/object-store/object.hpp b/src/realm/object-store/object.hpp index 77ed3afb6b5..a96df91acbc 100644 --- a/src/realm/object-store/object.hpp +++ b/src/realm/object-store/object.hpp @@ -127,7 +127,7 @@ class Object { * callback via `remove_callback`. */ NotificationToken add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array = std::nullopt) &; + KeyPathArray key_path_array = {}) &; template void set_column_value(StringData prop_name, ValueType&& value) diff --git a/src/realm/object-store/object_accessor.hpp b/src/realm/object-store/object_accessor.hpp index 0c6a88d8b86..e377ed9f50f 100644 --- a/src/realm/object-store/object_accessor.hpp +++ b/src/realm/object-store/object_accessor.hpp @@ -264,7 +264,7 @@ Object Object::create(ContextType& ctx, std::shared_ptr const& realm, Obj // considered a primary key by core, and so will need to be set. bool skip_primary = true; // If the input value is missing values for any of the properties we want to - // set the property to the default value for new objects, but leave it + // set the propery to the default value for new objects, but leave it // untouched for existing objects. bool created = false; diff --git a/src/realm/object-store/object_store.cpp b/src/realm/object-store/object_store.cpp index a33e1a807f9..8128f2d6b0e 100644 --- a/src/realm/object-store/object_store.cpp +++ b/src/realm/object-store/object_store.cpp @@ -875,6 +875,7 @@ void ObjectStore::apply_schema_changes(Transaction& group, uint64_t schema_versi } if (mode == SchemaMode::Manual) { + set_schema_keys(group, target_schema); if (migration_function) { migration_function(); } diff --git a/src/realm/object-store/results.cpp b/src/realm/object-store/results.cpp index b02d44f73bb..b23ff9dbfc7 100644 --- a/src/realm/object-store/results.cpp +++ b/src/realm/object-store/results.cpp @@ -1005,8 +1005,7 @@ void Results::prepare_async(ForCallback force) NO_THREAD_SAFETY_ANALYSIS _impl::RealmCoordinator::register_notifier(m_notifier); } -NotificationToken Results::add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array) & +NotificationToken Results::add_notification_callback(CollectionChangeCallback callback, KeyPathArray key_path_array) & { prepare_async(ForCallback{true}); return {m_notifier, m_notifier->add_callback(std::move(callback), std::move(key_path_array))}; diff --git a/src/realm/object-store/results.hpp b/src/realm/object-store/results.hpp index e5f90053c3d..e0b5c0f307e 100644 --- a/src/realm/object-store/results.hpp +++ b/src/realm/object-store/results.hpp @@ -304,7 +304,7 @@ class Results { * callback via `remove_callback`. */ NotificationToken add_notification_callback(CollectionChangeCallback callback, - std::optional key_path_array = std::nullopt) &; + KeyPathArray key_path_array = {}) &; // Returns whether the rows are guaranteed to be in table order. bool is_in_table_order() const; diff --git a/src/realm/object-store/schema.cpp b/src/realm/object-store/schema.cpp index 00e54d9f163..97a7422a919 100644 --- a/src/realm/object-store/schema.cpp +++ b/src/realm/object-store/schema.cpp @@ -313,10 +313,12 @@ void Schema::zip_matching(T&& a, U&& b, Func&& func) ++j; } } - for (; i < a.size(); ++i) + for (; i < a.size(); ++i) { func(&a[i], nullptr); - for (; j < b.size(); ++j) + } + for (; j < b.size(); ++j) { func(nullptr, &b[j]); + } } std::vector Schema::compare(Schema const& target_schema, SchemaMode mode, @@ -341,8 +343,10 @@ std::vector Schema::compare(Schema const& target_schema, SchemaMod // Modify columns zip_matching(target_schema, *this, [&](const ObjectSchema* target, const ObjectSchema* existing) { - if (target && existing) + if (target && existing) { ::compare(*existing, *target, changes); + } + else if (target && !orphans.count(target->name)) { // Target is a new table -- add all properties changes.emplace_back(schema_change::AddInitialProperties{target}); @@ -360,34 +364,38 @@ std::vector Schema::compare(Schema const& target_schema, SchemaMod return changes; } -void Schema::copy_keys_from(realm::Schema const& other, SchemaSubsetMode subset_mode) +void Schema::copy_keys_from(Schema const& other, bool is_schema_additive) { - std::vector other_classes; - zip_matching(*this, other, [&](ObjectSchema* existing, const ObjectSchema* other) { - if (subset_mode.include_types && !existing && other) - other_classes.push_back(other); + std::vector other_classes; + zip_matching(*this, other, [&](ObjectSchema const* existing, ObjectSchema const* other) { + if (is_schema_additive && !existing && other) { + other_classes.push_back(*other); + } if (!existing || !other) return; - - existing->table_key = other->table_key; - for (auto& current_prop : other->persisted_properties) { - if (auto target_prop = existing->property_for_name(current_prop.name)) { - target_prop->column_key = current_prop.column_key; - } - else if (subset_mode.include_properties) { - existing->persisted_properties.push_back(current_prop); - } - } + update_or_append_properties(const_cast(existing), other, is_schema_additive); }); if (!other_classes.empty()) { - reserve(size() + other_classes.size()); - for (auto other : other_classes) - push_back(*other); + insert(end(), other_classes.begin(), other_classes.end()); sort_schema(); } } +void Schema::update_or_append_properties(ObjectSchema* existing, const ObjectSchema* other, bool is_schema_additive) +{ + existing->table_key = other->table_key; + for (auto& current_prop : other->persisted_properties) { + auto target_prop = existing->property_for_name(current_prop.name); + if (target_prop) { + target_prop->column_key = current_prop.column_key; + } + else if (is_schema_additive) { + existing->persisted_properties.push_back(current_prop); + } + } +} + namespace realm { bool operator==(SchemaChange const& lft, SchemaChange const& rgt) noexcept { diff --git a/src/realm/object-store/schema.hpp b/src/realm/object-store/schema.hpp index 0a134bdd4cf..07f5b2b99de 100644 --- a/src/realm/object-store/schema.hpp +++ b/src/realm/object-store/schema.hpp @@ -114,43 +114,6 @@ enum class SchemaMode : uint8_t { Manual }; -// Options for how to handle the schema when the file has classes and/or -// properties not in the schema. -// -// Most schema modes allow the requested schema to be a subset of the actual -// schema of the Realm file. By default, any properties or object types not in -// the requested schema are simply ignored entirely and the Realm's in-memory -// schema will always exactly match the requested one. -struct SchemaSubsetMode { - // Add additional tables present in the Realm file to the schema. This is - // applicable to all schema modes except for Manual and ResetFile. - bool include_types : 1; - - // Add additional columns in the tables present in the Realm file to the - // object schema for those types. The additional properties are always - // added to the end of persisted_properties. This is only applicable to - // Additive and ReadOnly schema modes. - bool include_properties : 1; - - // The reported schema will always exactly match the requested one. - static const SchemaSubsetMode Strict; - // Additional object classes present in the Realm file are added to the - // requested schema, but all object types present in the requested schema - // will always exactly match even if there are additional columns in the - // tables. - static const SchemaSubsetMode AllClasses; - // Additional properties present in the Realm file are added to the - // requested schema, but tables not present in the schema are ignored. - static const SchemaSubsetMode AllProperties; - // Always report the complete schema. - static const SchemaSubsetMode Complete; -}; - -inline constexpr SchemaSubsetMode SchemaSubsetMode::Strict = {false, false}; -inline constexpr SchemaSubsetMode SchemaSubsetMode::AllClasses = {true, false}; -inline constexpr SchemaSubsetMode SchemaSubsetMode::AllProperties = {false, true}; -inline constexpr SchemaSubsetMode SchemaSubsetMode::Complete = {true, true}; - class Schema : private std::vector { private: @@ -188,7 +151,7 @@ class Schema : private std::vector { std::vector compare(Schema const&, SchemaMode = SchemaMode::Automatic, bool include_removals = false) const; - void copy_keys_from(Schema const&, SchemaSubsetMode subset_mode); + void copy_keys_from(Schema const&, bool is_schema_additive = false); friend bool operator==(Schema const&, Schema const&) noexcept; friend bool operator!=(Schema const& a, Schema const& b) noexcept @@ -208,6 +171,8 @@ class Schema : private std::vector { static void zip_matching(T&& a, U&& b, Func&& func); // sort all the classes by name in order to speed up find(StringData name) void sort_schema(); + // append missing properties and update matching properties for schema + void update_or_append_properties(ObjectSchema*, const ObjectSchema*, bool); }; namespace schema_change { diff --git a/src/realm/object-store/sectioned_results.cpp b/src/realm/object-store/sectioned_results.cpp index 39985676c3f..1fe4d9c9ded 100644 --- a/src/realm/object-store/sectioned_results.cpp +++ b/src/realm/object-store/sectioned_results.cpp @@ -306,7 +306,7 @@ size_t ResultsSection::size() } NotificationToken ResultsSection::add_notification_callback(SectionedResultsNotificatonCallback callback, - std::optional key_path_array) & + KeyPathArray key_path_array) & { return m_parent->add_notification_callback_for_section(m_key, std::move(callback), key_path_array); } @@ -445,14 +445,14 @@ ResultsSection SectionedResults::operator[](Mixed key) } NotificationToken SectionedResults::add_notification_callback(SectionedResultsNotificatonCallback callback, - std::optional key_path_array) & + KeyPathArray key_path_array) & { return m_results.add_notification_callback(SectionedResultsNotificationHandler(*this, std::move(callback)), std::move(key_path_array)); } NotificationToken SectionedResults::add_notification_callback_for_section( - Mixed section_key, SectionedResultsNotificatonCallback callback, std::optional key_path_array) + Mixed section_key, SectionedResultsNotificatonCallback callback, KeyPathArray key_path_array) { return m_results.add_notification_callback( SectionedResultsNotificationHandler(*this, std::move(callback), section_key), std::move(key_path_array)); diff --git a/src/realm/object-store/sectioned_results.hpp b/src/realm/object-store/sectioned_results.hpp index 62e89793b1e..e5a390c6929 100644 --- a/src/realm/object-store/sectioned_results.hpp +++ b/src/realm/object-store/sectioned_results.hpp @@ -81,7 +81,7 @@ class ResultsSection { * callback via `remove_callback`. */ NotificationToken add_notification_callback(SectionedResultsNotificatonCallback callback, - std::optional key_path_array = std::nullopt) &; + KeyPathArray key_path_array = {}) &; bool is_valid() const; @@ -139,7 +139,7 @@ class SectionedResults { * callback via `remove_callback`. */ NotificationToken add_notification_callback(SectionedResultsNotificatonCallback callback, - std::optional key_path_array = std::nullopt) &; + KeyPathArray key_path_array = {}) &; /// Return a new instance of SectionedResults that uses a snapshot of the underlying `Results`. /// The section key callback parameter will never be invoked. @@ -186,9 +186,9 @@ class SectionedResults { void calculate_sections_if_required() REQUIRES(m_mutex); void calculate_sections() REQUIRES(m_mutex); bool m_has_performed_initial_evalutation = false; - NotificationToken - add_notification_callback_for_section(Mixed section_key, SectionedResultsNotificatonCallback callback, - std::optional key_path_array = std::nullopt); + NotificationToken add_notification_callback_for_section(Mixed section_key, + SectionedResultsNotificatonCallback callback, + KeyPathArray key_path_array = {}); friend class realm::ResultsSection; Results m_results; diff --git a/src/realm/object-store/shared_realm.cpp b/src/realm/object-store/shared_realm.cpp index 06940d38b2b..e2538ad9bef 100644 --- a/src/realm/object-store/shared_realm.cpp +++ b/src/realm/object-store/shared_realm.cpp @@ -77,14 +77,10 @@ class CountGuard { Realm::Realm(Config config, util::Optional version, std::shared_ptr<_impl::RealmCoordinator> coordinator, MakeSharedTag) : m_config(std::move(config)) - , m_frozen_version(version) + , m_frozen_version(std::move(version)) , m_scheduler(m_config.scheduler) { - if (version) { - m_auto_refresh = false; - REALM_ASSERT(*version != VersionID()); - } - else if (!coordinator->get_cached_schema(m_schema, m_schema_version, m_schema_transaction_version)) { + if (!coordinator->get_cached_schema(m_schema, m_schema_version, m_schema_transaction_version)) { m_transaction = coordinator->begin_read(); read_schema_from_group_if_needed(); coordinator->cache_schema(m_schema, m_schema_version, m_schema_transaction_version); @@ -114,8 +110,9 @@ Group& Realm::read_group() Transaction& Realm::transaction() { verify_open(); - if (!m_transaction) + if (!m_transaction) { begin_read(m_frozen_version.value_or(VersionID{})); + } return *m_transaction; } @@ -163,7 +160,9 @@ SharedRealm Realm::get_shared_realm(Config config) SharedRealm Realm::get_frozen_realm(Config config, VersionID version) { auto coordinator = RealmCoordinator::get_coordinator(config.path); - return coordinator->get_realm(std::move(config), version); + SharedRealm realm = coordinator->get_realm(std::move(config), util::Optional(version)); + realm->set_auto_refresh(false); + return realm; } SharedRealm Realm::get_shared_realm(ThreadSafeReference ref, std::shared_ptr scheduler) @@ -219,7 +218,7 @@ sync::SubscriptionSet Realm::get_active_subscription_set() void Realm::set_schema(Schema const& reference, Schema schema) { m_dynamic_schema = false; - schema.copy_keys_from(reference, m_config.schema_subset_mode); + schema.copy_keys_from(reference, m_config.is_schema_additive()); m_schema = std::move(schema); notify_schema_changed(); } @@ -231,16 +230,15 @@ void Realm::read_schema_from_group_if_needed() if (m_schema.empty()) { m_schema_version = ObjectStore::get_schema_version(*m_transaction); m_schema = ObjectStore::schema_from_group(*m_transaction); - m_schema_transaction_version = m_transaction->get_version_of_current_transaction().version; } return; } - Group& group = read_group(); auto current_version = transaction().get_version_of_current_transaction().version; if (m_schema_transaction_version == current_version) return; + auto previous_transaction_version = m_schema_transaction_version; m_schema_transaction_version = current_version; m_schema_version = ObjectStore::get_schema_version(group); auto schema = ObjectStore::schema_from_group(group); @@ -251,16 +249,20 @@ void Realm::read_schema_from_group_if_needed() if (m_dynamic_schema) { if (m_schema == schema) { // The structure of the schema hasn't changed. Bring the table column indices up to date. - m_schema.copy_keys_from(schema, SchemaSubsetMode::Strict); + m_schema.copy_keys_from(schema); } else { // The structure of the schema has changed, so replace our copy of the schema. m_schema = std::move(schema); } } + else if (m_config.is_schema_additive() && m_schema_transaction_version < previous_transaction_version) { + // no verification of schema changes when opening a past version of the schema + m_schema.copy_keys_from(schema, m_config.is_schema_additive()); + } else { ObjectStore::verify_valid_external_changes(m_schema.compare(schema, m_config.schema_mode)); - m_schema.copy_keys_from(schema, m_config.schema_subset_mode); + m_schema.copy_keys_from(schema); } notify_schema_changed(); } @@ -337,12 +339,16 @@ Schema Realm::get_full_schema() do_refresh(); // If the user hasn't specified a schema previously then m_schema is always - // the full schema if it's been read - if (m_dynamic_schema && !m_schema.empty()) + // the full schema + if (m_dynamic_schema) { return m_schema; + } // Otherwise we may have a subset of the file's schema, so we need to get // the complete thing to calculate what changes to make + if (m_config.immutable()) + return ObjectStore::schema_from_group(read_group()); + Schema actual_schema; uint64_t actual_version; uint64_t version = -1; @@ -401,16 +407,8 @@ void Realm::update_schema(Schema schema, uint64_t version, MigrationFunction mig bool was_in_read_transaction = is_in_read_transaction(); Schema actual_schema = get_full_schema(); - - // Frozen Realms never modify the schema on disk and we just need to verify - // that the requested schema is a subset of what actually exists - if (m_frozen_version) { - ObjectStore::verify_valid_external_changes(schema.compare(actual_schema, m_config.schema_mode, true)); - set_schema(actual_schema, std::move(schema)); - return; - } - std::vector required_changes = actual_schema.compare(schema, m_config.schema_mode); + if (!schema_change_needs_write_transaction(schema, required_changes, version)) { if (!was_in_read_transaction) m_transaction = nullptr; @@ -446,13 +444,9 @@ void Realm::update_schema(Schema schema, uint64_t version, MigrationFunction mig cache_new_schema(); } - schema.copy_keys_from(actual_schema, m_config.schema_subset_mode); - uint64_t old_schema_version = m_schema_version; - bool additive = m_config.schema_mode == SchemaMode::AdditiveDiscovered || - m_config.schema_mode == SchemaMode::AdditiveExplicit || - m_config.schema_mode == SchemaMode::ReadOnly; - if (migration_function && !additive) { + bool is_schema_additive = m_config.is_schema_additive(); + if (migration_function && !is_schema_additive) { auto wrapper = [&] { auto config = m_config; config.schema_mode = SchemaMode::ReadOnly; @@ -482,7 +476,7 @@ void Realm::update_schema(Schema schema, uint64_t version, MigrationFunction mig else { ObjectStore::apply_schema_changes(transaction(), m_schema_version, schema, version, m_config.schema_mode, required_changes, m_config.automatically_handle_backlinks_in_migrations); - REALM_ASSERT_DEBUG(additive || + REALM_ASSERT_DEBUG(is_schema_additive || (required_changes = ObjectStore::schema_from_group(read_group()).compare(schema)).empty()); } @@ -528,7 +522,7 @@ void Realm::add_schema_change_handler() m_schema = *m_new_schema; } else { - m_schema.copy_keys_from(*m_new_schema, m_config.schema_subset_mode); + m_schema.copy_keys_from(*m_new_schema, m_config.is_schema_additive()); } notify_schema_changed(); }); @@ -536,17 +530,15 @@ void Realm::add_schema_change_handler() void Realm::cache_new_schema() { - if (is_closed()) { - return; + if (!is_closed()) { + auto new_version = transaction().get_version_of_current_transaction().version; + if (m_new_schema) + m_coordinator->cache_schema(std::move(*m_new_schema), m_schema_version, new_version); + else + m_coordinator->advance_schema_cache(m_schema_transaction_version, new_version); + m_schema_transaction_version = new_version; + m_new_schema = util::none; } - - auto new_version = transaction().get_version_of_current_transaction().version; - if (m_new_schema) - m_coordinator->cache_schema(std::move(*m_new_schema), m_schema_version, new_version); - else - m_coordinator->advance_schema_cache(m_schema_transaction_version, new_version); - m_schema_transaction_version = new_version; - m_new_schema = util::none; } void Realm::translate_schema_error() @@ -679,10 +671,10 @@ void Realm::enable_wait_for_change() bool Realm::wait_for_change() { - if (m_frozen_version || m_config.schema_mode == SchemaMode::Immutable) { + if (m_frozen_version) { return false; } - return m_transaction && m_coordinator->wait_for_change(m_transaction); + return m_transaction ? m_coordinator->wait_for_change(transaction_ref()) : false; } void Realm::wait_for_change_release() @@ -1319,18 +1311,12 @@ bool Realm::is_frozen() const SharedRealm Realm::freeze() { - read_group(); // Freezing requires a read transaction - return m_coordinator->freeze_realm(*this); -} - -void Realm::copy_schema_from(const Realm& source) -{ - REALM_ASSERT(is_frozen()); - REALM_ASSERT(m_frozen_version == source.read_transaction_version()); - m_schema = source.m_schema; - m_schema_version = source.m_schema_version; - m_schema_transaction_version = m_frozen_version->version; - m_dynamic_schema = false; + auto config = m_config; + auto version = read_transaction_version(); + config.scheduler = util::Scheduler::make_frozen(version); + auto frozen_realm = Realm::get_frozen_realm(std::move(config), version); + frozen_realm->set_schema(frozen_realm->m_schema, m_schema); + return frozen_realm; } void Realm::close() diff --git a/src/realm/object-store/shared_realm.hpp b/src/realm/object-store/shared_realm.hpp index 58cf7e8d478..19c557ea70f 100644 --- a/src/realm/object-store/shared_realm.hpp +++ b/src/realm/object-store/shared_realm.hpp @@ -104,7 +104,6 @@ struct RealmConfig { bool in_memory = false; SchemaMode schema_mode = SchemaMode::Automatic; - SchemaSubsetMode schema_subset_mode = SchemaSubsetMode::Strict; // Optional schema for the file. // If the schema and schema version are supplied, update_schema() is @@ -136,6 +135,11 @@ struct RealmConfig { { return schema_mode == SchemaMode::ReadOnly; } + bool is_schema_additive() const + { + return schema_mode == SchemaMode::AdditiveExplicit || schema_mode == SchemaMode::AdditiveDiscovered || + schema_mode == SchemaMode::ReadOnly; + } // If false, always return a new Realm instance, and don't return // that Realm instance for other requests for a cached Realm. Useful @@ -473,11 +477,6 @@ class Realm : public std::enable_shared_from_this { realm.run_writes(); } - static void copy_schema(Realm& target_realm, const Realm& source_realm) - { - target_realm.copy_schema_from(source_realm); - } - // CollectionNotifier needs to be able to access the owning // coordinator to wake up the worker thread when a callback is // added, and coordinators need to be able to get themselves from a Realm @@ -557,7 +556,6 @@ class Realm : public std::enable_shared_from_this { void cache_new_schema(); void translate_schema_error(); void notify_schema_changed(); - void copy_schema_from(const Realm&); Transaction& transaction(); Transaction& transaction() const; diff --git a/src/realm/object-store/sync/app.cpp b/src/realm/object-store/sync/app.cpp index 104c20859e8..d05238501c8 100644 --- a/src/realm/object-store/sync/app.cpp +++ b/src/realm/object-store/sync/app.cpp @@ -262,7 +262,6 @@ void App::configure(const SyncClientConfig& sync_client_config) auto sync_route = make_sync_route(m_app_route); m_sync_manager->configure(shared_from_this(), sync_route, sync_client_config); if (auto metadata = m_sync_manager->app_metadata()) { - // If there is app metadata stored, then update the hostname/syncroute using that info update_hostname(metadata); } } @@ -311,7 +310,7 @@ std::string App::make_sync_route(const std::string& http_app_route) void App::update_hostname(const util::Optional& metadata) { - // Update url components based on new hostname value from the app metadata + // Update url components based on new hostname value if (metadata) { update_hostname(metadata->hostname, metadata->ws_hostname); } @@ -319,9 +318,8 @@ void App::update_hostname(const util::Optional& metadata) void App::update_hostname(const std::string& hostname, const Optional& ws_hostname) { - // Update url components based on new hostname (and optional websocket hostname) values + // Update url components based on new hostname value log_debug("App: update_hostname: %1 | %2", hostname, ws_hostname); - REALM_ASSERT(m_sync_manager); std::lock_guard lock(*m_route_mutex); m_base_route = (hostname.length() > 0 ? hostname : default_base_url) + base_path; std::string this_app_path = app_path + "/" + m_config.app_id; @@ -330,7 +328,7 @@ void App::update_hostname(const std::string& hostname, const Optionallength() > 0) { m_sync_manager->set_sync_route(*ws_hostname + base_path + this_app_path + sync_path); } - else { + else if (m_sync_manager) { m_sync_manager->set_sync_route(make_sync_route(m_app_route)); } } @@ -829,8 +827,8 @@ void App::init_app_metadata(UniqueFunction&)>&& co { std::string route; - if (!new_hostname && (m_sync_manager->app_metadata() || m_location_updated)) { - // Skip if the app_metadata/location data has already been initialized and a new hostname is not provided + if (!new_hostname && m_sync_manager->app_metadata()) { + // Skip if the app_metadata has already been initialized and a new hostname is not provided return completion(util::none); // early return } else { @@ -856,17 +854,11 @@ void App::init_app_metadata(UniqueFunction&)>&& co auto ws_hostname = get(json, "ws_hostname"); auto deployment_model = get(json, "deployment_model"); auto location = get(json, "location"); - if (self->m_sync_manager->perform_metadata_update([&](SyncMetadataManager& manager) { - manager.set_app_metadata(deployment_model, location, hostname, ws_hostname); - })) { - // Update the hostname and sync route using the new app metadata info - self->update_hostname(self->m_sync_manager->app_metadata()); - } - else { - // No metadata in use, update the hostname and sync route directly - self->update_hostname(hostname, ws_hostname); - } - self->m_location_updated = true; + self->m_sync_manager->perform_metadata_update([&](SyncMetadataManager& manager) { + manager.set_app_metadata(deployment_model, location, hostname, ws_hostname); + }); + + self->update_hostname(self->m_sync_manager->app_metadata()); } catch (const AppError&) { // Pass the response back to completion diff --git a/src/realm/object-store/sync/app.hpp b/src/realm/object-store/sync/app.hpp index 194e2eb9a09..ee41c2b6aaf 100644 --- a/src/realm/object-store/sync/app.hpp +++ b/src/realm/object-store/sync/app.hpp @@ -386,7 +386,6 @@ class App : public std::enable_shared_from_this, std::string m_app_route; std::string m_auth_route; uint64_t m_request_timeout_ms; - bool m_location_updated = false; std::shared_ptr m_sync_manager; std::shared_ptr m_logger_ptr; diff --git a/src/realm/object-store/sync/async_open_task.cpp b/src/realm/object-store/sync/async_open_task.cpp index c5960f58dd3..b08fdbef8d5 100644 --- a/src/realm/object-store/sync/async_open_task.cpp +++ b/src/realm/object-store/sync/async_open_task.cpp @@ -94,7 +94,7 @@ void AsyncOpenTask::cancel() // thus deadlocking. if (session) { // Does a better way exists for canceling the download? - session->force_close(); + session->log_out(); } } diff --git a/src/realm/object-store/sync/impl/sync_client.hpp b/src/realm/object-store/sync/impl/sync_client.hpp index 4e3be582242..f43b2ae20ef 100644 --- a/src/realm/object-store/sync/impl/sync_client.hpp +++ b/src/realm/object-store/sync/impl/sync_client.hpp @@ -19,9 +19,9 @@ #ifndef REALM_OS_SYNC_CLIENT_HPP #define REALM_OS_SYNC_CLIENT_HPP +#include + #include -#include -#include #include #include @@ -39,20 +39,15 @@ namespace _impl { struct SyncClient { SyncClient(const std::shared_ptr& logger, SyncClientConfig const& config, std::weak_ptr weak_sync_manager) - : m_socket_provider([&]() -> std::shared_ptr { - if (config.socket_provider) { - return config.socket_provider; - } - auto user_agent = util::format("RealmSync/%1 (%2) %3 %4", REALM_VERSION_STRING, util::get_platform_info(), - config.user_agent_binding_info, config.user_agent_application_info); - return std::make_shared(logger, std::move(user_agent)); - }()) - , m_client([&] { + : m_client([&] { sync::Client::Config c; c.logger = logger; - c.socket_provider = m_socket_provider; + c.socket_provider = config.socket_provider; c.reconnect_mode = config.reconnect_mode; c.one_connection_per_session = !config.multiplex_sessions; + /// DEPRECATED - Will be removed in a future release + c.user_agent_application_info = + util::format("%1 %2", config.user_agent_binding_info, config.user_agent_application_info); // Only set the timeouts if they have sensible values if (config.timeouts.connect_timeout >= 1000) @@ -70,6 +65,23 @@ struct SyncClient { }()) , m_logger_ptr(logger) , m_logger(*m_logger_ptr) + , m_thread([this] { + if (g_binding_callback_thread_observer) { + g_binding_callback_thread_observer->did_create_thread(); + auto will_destroy_thread = util::make_scope_exit([&]() noexcept { + g_binding_callback_thread_observer->will_destroy_thread(); + }); + try { + m_client.run(); // Throws + } + catch (std::exception const& e) { + g_binding_callback_thread_observer->handle_error(e); + } + } + else { + m_client.run(); // Throws + } + }) // Throws #if NETWORK_REACHABILITY_AVAILABLE , m_reachability_observer(none, [weak_sync_manager](const NetworkReachabilityStatus status) { if (status != NotReachable) { @@ -96,6 +108,8 @@ struct SyncClient { void stop() { m_client.stop(); + if (m_thread.joinable()) + m_thread.join(); } std::unique_ptr make_session(std::shared_ptr db, @@ -116,13 +130,16 @@ struct SyncClient { m_client.wait_for_session_terminations_or_client_stopped(); } - ~SyncClient() {} + ~SyncClient() + { + stop(); + } private: - std::shared_ptr m_socket_provider; sync::Client m_client; std::shared_ptr m_logger_ptr; util::Logger& m_logger; + std::thread m_thread; #if NETWORK_REACHABILITY_AVAILABLE NetworkReachabilityObserver m_reachability_observer; #endif diff --git a/src/realm/object-store/sync/sync_manager.cpp b/src/realm/object-store/sync/sync_manager.cpp index 83a508c6f9b..a91584c1919 100644 --- a/src/realm/object-store/sync/sync_manager.cpp +++ b/src/realm/object-store/sync/sync_manager.cpp @@ -782,7 +782,7 @@ void SyncManager::close_all_sessions() } for (auto& [_, session] : sessions) { - session->force_close(); + session->log_out(); } get_sync_client().wait_for_session_terminations(); diff --git a/src/realm/object-store/sync/sync_session.cpp b/src/realm/object-store/sync/sync_session.cpp index 25ead85a676..f539090f65b 100644 --- a/src/realm/object-store/sync/sync_session.cpp +++ b/src/realm/object-store/sync/sync_session.cpp @@ -103,30 +103,6 @@ void SyncSession::become_active() } } -void SyncSession::restart_session() -{ - util::CheckedLockGuard lock(m_state_mutex); - // Nothing to do if the sync session is currently paused - // It will be resumed when resume() is called - if (m_state == State::Paused) - return; - - // Go straight to inactive so the progress completion waiters will - // continue to wait until the session restarts and completes the - // upload/download sync - m_state = State::Inactive; - - if (m_session) { - m_session.reset(); - } - - // Create a new session and re-register the completion callbacks - // The latest server path will be retrieved from sync_manager when - // the new session is created by create_sync_session() in become - // active. - become_active(); -} - void SyncSession::become_dying(util::CheckedUniqueLock lock) { REALM_ASSERT(m_state != State::Dying); @@ -156,26 +132,6 @@ void SyncSession::become_inactive(util::CheckedUniqueLock lock, std::error_code REALM_ASSERT(m_state != State::Inactive); m_state = State::Inactive; - do_become_inactive(std::move(lock), ec); -} - -void SyncSession::become_paused(util::CheckedUniqueLock lock) -{ - REALM_ASSERT(m_state != State::Paused); - auto old_state = m_state; - m_state = State::Paused; - - // Nothing to do if we're already inactive besides update the state. - if (old_state == State::Inactive) { - m_state_mutex.unlock(lock); - return; - } - - do_become_inactive(std::move(lock), {}); -} - -void SyncSession::do_become_inactive(util::CheckedUniqueLock lock, std::error_code ec) -{ // Manually set the disconnected state. Sync would also do this, but // since the underlying SyncSession object already have been destroyed, // we are not able to get the callback. @@ -257,9 +213,9 @@ static bool check_for_redirect_response(const app::AppError& error) } util::UniqueFunction)> -SyncSession::handle_refresh(const std::shared_ptr& session, bool restart_session) +SyncSession::handle_refresh(const std::shared_ptr& session) { - return [session, restart_session](util::Optional error) { + return [session](util::Optional error) { auto session_user = session->user(); if (!session_user) { util::CheckedUniqueLock lock(session->m_state_mutex); @@ -309,16 +265,7 @@ SyncSession::handle_refresh(const std::shared_ptr& session, bool re } } else { - // If the session needs to be restarted, then restart the session now - // The latest access token and server url will be pulled from the sync - // manager when the new session is started. - if (restart_session) { - session->restart_session(); - } - // Otherwise, update the access token and reconnect - else { - session->update_access_token(session_user->access_token()); - } + session->update_access_token(session_user->access_token()); } }; } @@ -418,9 +365,7 @@ void SyncSession::download_fresh_realm(sync::ProtocolErrorInfo::Action server_re auto mode = config(&SyncConfig::client_resync_mode); if (mode == ClientResyncMode::Recover) { handle_fresh_realm_downloaded( - nullptr, - {ErrorCodes::RuntimeError, - "A client reset is required but the server does not permit recovery for this client"}, + nullptr, {"A client reset is required but the server does not permit recovery for this client"}, server_requests_action); } } @@ -454,10 +399,10 @@ void SyncSession::download_fresh_realm(sync::ProtocolErrorInfo::Action server_re db = DB::create(sync::make_client_replication(), fresh_path, options); } } - catch (...) { + catch (std::exception const& e) { // Failed to open the fresh path after attempting to delete it, so we // just can't do automatic recovery. - handle_fresh_realm_downloaded(nullptr, exception_to_status(), server_requests_action); + handle_fresh_realm_downloaded(nullptr, std::string(e.what()), server_requests_action); return; } @@ -465,25 +410,25 @@ void SyncSession::download_fresh_realm(sync::ProtocolErrorInfo::Action server_re if (m_state != State::Active) { return; } - std::shared_ptr fresh_sync_session; + std::shared_ptr sync_session; { util::CheckedLockGuard config_lock(m_config_mutex); RealmConfig config = m_config; - config.path = fresh_path; // deep copy the sync config so we don't modify the live session's config config.sync_config = std::make_shared(*m_config.sync_config); + config.sync_config->stop_policy = SyncSessionStopPolicy::Immediately; config.sync_config->client_resync_mode = ClientResyncMode::Manual; - fresh_sync_session = m_sync_manager->get_session(db, config); + sync_session = create(m_client, db, config, m_sync_manager); auto& history = static_cast(*db->get_replication()); // the fresh Realm may apply writes to this db after it has outlived its sync session // the writes are used to generate a changeset for recovery, but are never committed history.set_write_validator_factory({}); } - fresh_sync_session->assert_mutex_unlocked(); + sync_session->assert_mutex_unlocked(); if (m_flx_subscription_store) { sync::SubscriptionSet active = m_flx_subscription_store->get_active(); - auto fresh_sub_store = fresh_sync_session->get_flx_subscription_store(); + auto fresh_sub_store = sync_session->get_flx_subscription_store(); REALM_ASSERT(fresh_sub_store); auto fresh_mut_sub = fresh_sub_store->get_latest().make_mutable_copy(); fresh_mut_sub.import(active); @@ -493,41 +438,37 @@ void SyncSession::download_fresh_realm(sync::ProtocolErrorInfo::Action server_re .get_async([=, weak_self = weak_from_this()](StatusWith s) { // Keep the sync session alive while it's downloading, but then close // it immediately - fresh_sync_session->force_close(); + sync_session->close(); if (auto strong_self = weak_self.lock()) { if (s.is_ok()) { - strong_self->handle_fresh_realm_downloaded(db, Status::OK(), server_requests_action); + strong_self->handle_fresh_realm_downloaded(db, none, server_requests_action); } else { - strong_self->handle_fresh_realm_downloaded(nullptr, s.get_status(), server_requests_action); + strong_self->handle_fresh_realm_downloaded(nullptr, s.get_status().reason(), + server_requests_action); } } }); } else { // pbs - fresh_sync_session->wait_for_download_completion([=, weak_self = weak_from_this()](std::error_code ec) { + sync_session->wait_for_download_completion([=, weak_self = weak_from_this()](std::error_code ec) { // Keep the sync session alive while it's downloading, but then close // it immediately - fresh_sync_session->force_close(); + sync_session->close(); if (auto strong_self = weak_self.lock()) { - if (ec == util::error::operation_aborted) { - strong_self->handle_fresh_realm_downloaded(nullptr, {ErrorCodes::OperationAborted, ec.message()}, - server_requests_action); - } - else if (ec) { - strong_self->handle_fresh_realm_downloaded(nullptr, {ErrorCodes::RuntimeError, ec.message()}, - server_requests_action); + if (ec) { + strong_self->handle_fresh_realm_downloaded(nullptr, ec.message(), server_requests_action); } else { - strong_self->handle_fresh_realm_downloaded(db, Status::OK(), server_requests_action); + strong_self->handle_fresh_realm_downloaded(db, none, server_requests_action); } } }); } - fresh_sync_session->revive_if_needed(); + sync_session->revive_if_needed(); } -void SyncSession::handle_fresh_realm_downloaded(DBRef db, Status status, +void SyncSession::handle_fresh_realm_downloaded(DBRef db, util::Optional error_message, sync::ProtocolErrorInfo::Action server_requests_action) { util::CheckedUniqueLock lock(m_state_mutex); @@ -538,10 +479,7 @@ void SyncSession::handle_fresh_realm_downloaded(DBRef db, Status status, // - unable to write the fresh copy to the file system // - during download of the fresh copy, the fresh copy itself is reset // - in FLX mode there was a problem fulfilling the previously active subscription - if (!status.is_ok()) { - if (status == ErrorCodes::OperationAborted) { - return; - } + if (error_message) { lock.unlock(); if (m_flx_subscription_store) { // In DiscardLocal mode, only the active subscription set is preserved @@ -550,13 +488,12 @@ void SyncSession::handle_fresh_realm_downloaded(DBRef db, Status status, auto mut_sub = m_flx_subscription_store->get_active().make_mutable_copy(); m_flx_subscription_store->supercede_all_except(mut_sub); mut_sub.update_state(sync::SubscriptionSet::State::Error, - util::make_optional(status.reason())); + util::make_optional(*error_message)); std::move(mut_sub).commit(); } const bool is_fatal = true; SyncError synthetic(make_error_code(sync::Client::Error::auto_client_reset_failure), - util::format("A fatal error occured during client reset: '%1'", status.reason()), - is_fatal); + util::format("A fatal error occured during client reset: '%1'", error_message), is_fatal); handle_error(synthetic); return; } @@ -695,14 +632,10 @@ void SyncSession::handle_error(SyncError error) // is disabled. In this scenario we attempt an automatic token refresh and if that succeeds continue as // normal. If the refresh request also fails with 401 then we need to stop retrying and pass along the error; // see handle_refresh(). - if (error_code.category() == sync::websocket::websocket_close_status_category()) { - bool restart_session = error_code.value() == ErrorCodes::WebSocket_MovedPermanently; - if (restart_session || error_code.value() == ErrorCodes::WebSocket_Unauthorized || - error_code.value() == ErrorCodes::WebSocket_AbnormalClosure) { - if (auto u = user()) { - u->refresh_custom_data(handle_refresh(shared_from_this(), restart_session)); - return; - } + if (error_code == sync::websocket::make_error_code(sync::websocket::Error::bad_response_401_unauthorized)) { + if (auto u = user()) { + u->refresh_custom_data(handle_refresh(shared_from_this())); + return; } } // Unrecognized error code. @@ -720,7 +653,7 @@ void SyncSession::handle_error(SyncError error) // Dont't bother invoking m_config.error_handler if the sync is inactive. // It does not make sense to call the handler when the session is closed. - if (m_state == State::Inactive || m_state == State::Paused) { + if (m_state == State::Inactive) { return; } @@ -767,36 +700,45 @@ void SyncSession::handle_progress_update(uint64_t downloaded, uint64_t downloada m_progress_notifier.update(downloaded, downloadable, uploaded, uploadable, download_version, snapshot_version); } -static sync::Session::Config::ClientReset make_client_reset_config(RealmConfig session_config, DBRef&& fresh_copy, +static sync::Session::Config::ClientReset make_client_reset_config(RealmConfig& session_config, DBRef&& fresh_copy, bool recovery_is_allowed) { - REALM_ASSERT(session_config.sync_config->client_resync_mode != ClientResyncMode::Manual); - + RealmConfig copy_config = session_config; + copy_config.sync_config = std::make_shared(*session_config.sync_config); // deep copy sync::Session::Config::ClientReset config; + REALM_ASSERT(session_config.sync_config->client_resync_mode != ClientResyncMode::Manual); config.mode = session_config.sync_config->client_resync_mode; - config.fresh_copy = std::move(fresh_copy); - config.recovery_is_allowed = recovery_is_allowed; - - session_config.sync_config = std::make_shared(*session_config.sync_config); // deep copy - session_config.scheduler = nullptr; - if (session_config.sync_config->notify_after_client_reset) { - config.notify_after_client_reset = [config = session_config](VersionID previous_version, bool did_recover) { + if (copy_config.sync_config->notify_after_client_reset) { + config.notify_after_client_reset = [config = copy_config](std::string local_path, VersionID previous_version, + bool did_recover) { + REALM_ASSERT(local_path == config.path); auto local_coordinator = RealmCoordinator::get_coordinator(config); REALM_ASSERT(local_coordinator); + auto local_config = local_coordinator->get_config(); ThreadSafeReference active_after = local_coordinator->get_unbound_realm(); - SharedRealm frozen_before = local_coordinator->get_realm(config, previous_version); + local_config.scheduler = nullptr; + SharedRealm frozen_before = local_coordinator->get_realm(local_config, previous_version); REALM_ASSERT(frozen_before); REALM_ASSERT(frozen_before->is_frozen()); - config.sync_config->notify_after_client_reset(std::move(frozen_before), std::move(active_after), - did_recover); + config.sync_config->notify_after_client_reset(frozen_before, std::move(active_after), did_recover); }; } - if (session_config.sync_config->notify_before_client_reset) { - config.notify_before_client_reset = [config = session_config](VersionID version) { - config.sync_config->notify_before_client_reset(Realm::get_frozen_realm(config, version)); + if (copy_config.sync_config->notify_before_client_reset) { + config.notify_before_client_reset = [config = copy_config](std::string local_path) { + REALM_ASSERT(local_path == config.path); + auto local_coordinator = RealmCoordinator::get_coordinator(config); + REALM_ASSERT(local_coordinator); + auto local_config = local_coordinator->get_config(); + local_config.scheduler = nullptr; + SharedRealm frozen_local = local_coordinator->get_realm(local_config, VersionID()); + REALM_ASSERT(frozen_local); + REALM_ASSERT(frozen_local->is_frozen()); + config.sync_config->notify_before_client_reset(frozen_local); }; } + config.fresh_copy = std::move(fresh_copy); + config.recovery_is_allowed = recovery_is_allowed; return config; } @@ -940,7 +882,6 @@ void SyncSession::nonsync_transact_notify(sync::version_type version) break; case State::Dying: case State::Inactive: - case State::Paused: break; } } @@ -951,12 +892,25 @@ void SyncSession::revive_if_needed() switch (m_state) { case State::Active: case State::WaitingForAccessToken: - case State::Paused: return; case State::Dying: - case State::Inactive: - do_revive(std::move(lock)); + case State::Inactive: { + // Revive. + auto u = user(); + if (!u || !u->access_token_refresh_required()) { + become_active(); + return; + } + + become_waiting_for_access_token(); + // Release the lock for SDKs with a single threaded + // networking implementation such as our test suite + // so that the update can trigger a state change from + // the completion handler. + lock.unlock(); + initiate_access_token_refresh(); break; + } } } @@ -970,12 +924,11 @@ void SyncSession::handle_reconnect() case State::Dying: case State::Inactive: case State::WaitingForAccessToken: - case State::Paused: break; } } -void SyncSession::force_close() +void SyncSession::log_out() { util::CheckedUniqueLock lock(m_state_mutex); switch (m_state) { @@ -985,59 +938,10 @@ void SyncSession::force_close() become_inactive(std::move(lock)); break; case State::Inactive: - case State::Paused: break; } } -void SyncSession::pause() -{ - util::CheckedUniqueLock lock(m_state_mutex); - switch (m_state) { - case State::Active: - case State::Dying: - case State::WaitingForAccessToken: - case State::Inactive: - become_paused(std::move(lock)); - break; - case State::Paused: - break; - } -} - -void SyncSession::resume() -{ - util::CheckedUniqueLock lock(m_state_mutex); - switch (m_state) { - case State::Active: - case State::WaitingForAccessToken: - return; - case State::Paused: - case State::Dying: - case State::Inactive: - do_revive(std::move(lock)); - break; - } -} - -void SyncSession::do_revive(util::CheckedUniqueLock&& lock) -{ - auto u = user(); - if (!u || !u->access_token_refresh_required()) { - become_active(); - m_state_mutex.unlock(lock); - return; - } - - become_waiting_for_access_token(); - // Release the lock for SDKs with a single threaded - // networking implementation such as our test suite - // so that the update can trigger a state change from - // the completion handler. - m_state_mutex.unlock(lock); - initiate_access_token_refresh(); -} - void SyncSession::close() { util::CheckedUniqueLock lock(m_state_mutex); @@ -1066,10 +970,6 @@ void SyncSession::close(util::CheckedUniqueLock lock) case State::Dying: m_state_mutex.unlock(lock); break; - case State::Paused: - // The paused state is sticky, so we don't transition to inactive here if we're already paused. - m_state_mutex.unlock(lock); - break; case State::Inactive: { if (m_sync_manager) { m_sync_manager->unregister_session(m_db->get_path()); @@ -1094,7 +994,7 @@ void SyncSession::shutdown_and_wait() // Realm file to be closed. This works so long as this SyncSession object remains in the // `inactive` state after the invocation of shutdown_and_wait(). util::CheckedUniqueLock lock(m_state_mutex); - if (m_state != State::Inactive && m_state != State::Paused) { + if (m_state != State::Inactive) { become_inactive(std::move(lock)); } } @@ -1207,24 +1107,11 @@ const std::shared_ptr& SyncSession::get_flx_subscriptio return m_flx_subscription_store; } -sync::SaltedFileIdent SyncSession::get_file_ident() const -{ - auto repl = m_db->get_replication(); - REALM_ASSERT(repl); - REALM_ASSERT(dynamic_cast(repl)); - - sync::SaltedFileIdent ret; - sync::version_type unused_version; - sync::SyncProgress unused_progress; - static_cast(repl)->get_history().get_status(unused_version, ret, unused_progress); - return ret; -} - void SyncSession::update_configuration(SyncConfig new_config) { while (true) { util::CheckedUniqueLock state_lock(m_state_mutex); - if (m_state != State::Inactive && m_state != State::Paused) { + if (m_state != State::Inactive) { // Changing the state releases the lock, which means that by the // time we reacquire the lock the state may have changed again // (either due to one of the callbacks being invoked or another @@ -1235,7 +1122,7 @@ void SyncSession::update_configuration(SyncConfig new_config) } util::CheckedUniqueLock config_lock(m_config_mutex); - REALM_ASSERT(m_state == State::Inactive || m_state == State::Paused); + REALM_ASSERT(m_state == State::Inactive); REALM_ASSERT(!m_session); REALM_ASSERT(m_config.sync_config->user == new_config.user); m_config.sync_config = std::make_shared(std::move(new_config)); diff --git a/src/realm/object-store/sync/sync_session.hpp b/src/realm/object-store/sync/sync_session.hpp index 3c57c72fdb3..1459e7abf3a 100644 --- a/src/realm/object-store/sync/sync_session.hpp +++ b/src/realm/object-store/sync/sync_session.hpp @@ -107,7 +107,6 @@ class SyncSession : public std::enable_shared_from_this { Dying, Inactive, WaitingForAccessToken, - Paused, }; enum class ConnectionState { @@ -176,40 +175,17 @@ class SyncSession : public std::enable_shared_from_this { // Specifically: // If the sync session is currently `Dying`, ask it to stay alive instead. // If the sync session is currently `Inactive`, recreate it. - // If the sync session is currently `Paused`, do nothing - call resume() instead. // Otherwise, a no-op. void revive_if_needed() REQUIRES(!m_state_mutex, !m_config_mutex); // Perform any actions needed in response to regaining network connectivity. void handle_reconnect() REQUIRES(!m_state_mutex); - // Inform the sync session that it should close. This will respect the stop policy specified in - // the SyncConfig, so its possible the session will remain open either until all pending local - // changes are uploaded or possibly forever. + // Inform the sync session that it should close. void close() REQUIRES(!m_state_mutex, !m_config_mutex, !m_connection_state_mutex); - // Inform the sync session that it should close immediately, regardless of the stop policy. - // The session may resume after calling this if a new Realm is opened for the underlying DB - // of the SyncSession. Use pause() to close the sync session until you want to explicitly - // resume it. - void force_close() REQUIRES(!m_state_mutex, !m_connection_state_mutex); - - // Closes the sync session so that it will not resume until resume() is called. - void pause() REQUIRES(!m_state_mutex, !m_connection_state_mutex); - - // Resumes the sync session after it was paused by calling pause(). If the sync session is inactive - // for any other reason this will also resume it. - void resume() REQUIRES(!m_state_mutex, !m_config_mutex); - - // Drop the current session and restart a new one from scratch using the latest configuration in - // the sync manager. Used to respond to redirect responses from the server when the deployment - // model has changed while the user is logged in and a session is active. - // If this sync session is currently paused, a new session will not be started until resume() is - // called. - // NOTE: This method ignores the current stop policy and closes the current session immediately, - // since a new session will be created as part of this call. The new session will adhere to - // the stop policy if it is manually closed. - void restart_session() REQUIRES(!m_state_mutex, !m_connection_state_mutex, !m_config_mutex); + // Inform the sync session that it should log out. + void log_out() REQUIRES(!m_state_mutex, !m_connection_state_mutex); // Shut down the synchronization session (sync::Session) and wait for the Realm file to no // longer be open on behalf of it. @@ -304,11 +280,6 @@ class SyncSession : public std::enable_shared_from_this { { return session.send_test_command(std::move(request)); } - - static sync::SaltedFileIdent get_file_ident(SyncSession& session) - { - return session.get_file_ident(); - } }; private: @@ -356,13 +327,13 @@ class SyncSession : public std::enable_shared_from_this { std::shared_ptr sync_manager() const REQUIRES(!m_state_mutex); static util::UniqueFunction)> - handle_refresh(const std::shared_ptr&, bool = false); + handle_refresh(const std::shared_ptr&); SyncSession(_impl::SyncClient&, std::shared_ptr, const RealmConfig&, SyncManager* sync_manager); void download_fresh_realm(sync::ProtocolErrorInfo::Action server_requests_action) REQUIRES(!m_config_mutex, !m_state_mutex, !m_connection_state_mutex); - void handle_fresh_realm_downloaded(DBRef db, Status status, + void handle_fresh_realm_downloaded(DBRef db, util::Optional error_message, sync::ProtocolErrorInfo::Action server_requests_action) REQUIRES(!m_state_mutex, !m_config_mutex, !m_connection_state_mutex); void handle_error(SyncError) REQUIRES(!m_state_mutex, !m_config_mutex, !m_connection_state_mutex); @@ -386,22 +357,11 @@ class SyncSession : public std::enable_shared_from_this { void become_dying(util::CheckedUniqueLock) RELEASE(m_state_mutex) REQUIRES(!m_connection_state_mutex); void become_inactive(util::CheckedUniqueLock, std::error_code ec = {}) RELEASE(m_state_mutex) REQUIRES(!m_connection_state_mutex); - void become_paused(util::CheckedUniqueLock) RELEASE(m_state_mutex) REQUIRES(!m_connection_state_mutex); void become_waiting_for_access_token() REQUIRES(m_state_mutex); - // do_become_inactive is called from both become_paused()/become_inactive() and does all the steps to - // shutdown and cleanup the sync session besides setting m_state. - void do_become_inactive(util::CheckedUniqueLock, std::error_code ec) RELEASE(m_state_mutex) - REQUIRES(!m_connection_state_mutex); - // do_revive is called from both revive_if_needed() and resume(). It does all the steps to transition - // from a state that is not Active to Active. - void do_revive(util::CheckedUniqueLock&& lock) RELEASE(m_state_mutex) REQUIRES(!m_config_mutex); - void add_completion_callback(util::UniqueFunction callback, ProgressDirection direction) REQUIRES(m_state_mutex); - sync::SaltedFileIdent get_file_ident() const; - util::Future send_test_command(std::string body) REQUIRES(!m_state_mutex); std::function m_sync_transact_callback GUARDED_BY(m_state_mutex); diff --git a/src/realm/object-store/sync/sync_user.cpp b/src/realm/object-store/sync/sync_user.cpp index 3aff93a46a7..69f4ef3cbca 100644 --- a/src/realm/object-store/sync/sync_user.cpp +++ b/src/realm/object-store/sync/sync_user.cpp @@ -355,7 +355,7 @@ void SyncUser::log_out() // logged back in, they will automatically be reactivated. for (auto& [path, weak_session] : m_sessions) { if (auto ptr = weak_session.lock()) { - ptr->force_close(); + ptr->log_out(); m_waiting_sessions[path] = std::move(ptr); } } diff --git a/src/realm/set.cpp b/src/realm/set.cpp index ac134c85ada..148794d0f4f 100644 --- a/src/realm/set.cpp +++ b/src/realm/set.cpp @@ -320,47 +320,4 @@ void set_sorted_indices(size_t sz, std::vector& indices, bool ascending) } } -template -static bool partition_points(const Set& set, std::vector& indices, Iterator& first_string, - Iterator& first_binary, Iterator& end) -{ - first_string = std::partition_point(indices.begin(), indices.end(), [&](size_t i) { - return set.get(i).is_type(type_Bool, type_Int, type_Float, type_Double, type_Decimal); - }); - if (first_string == indices.end() || !set.get(*first_string).is_type(type_String)) - return false; - first_binary = std::partition_point(first_string + 1, indices.end(), [&](size_t i) { - return set.get(i).is_type(type_String); - }); - if (first_binary == indices.end() || !set.get(*first_binary).is_type(type_Binary)) - return false; - end = std::partition_point(first_binary + 1, indices.end(), [&](size_t i) { - return set.get(i).is_type(type_Binary); - }); - return true; -} - -template <> -void Set::sort(std::vector& indices, bool ascending) const -{ - set_sorted_indices(size(), indices, true); - - // The on-disk order is bool -> numbers -> string -> binary -> others - // We want to merge the string and binary sections to match the sort order - // of other collections. To do this we find the three partition points - // where the first string occurs, the first binary occurs, and the first - // non-binary after binaries occurs. If there's no strings or binaries we - // don't have to do anything. If they're both non-empty, we perform an - // in-place merge on the strings and binaries. - std::vector::iterator first_string, first_binary, end; - if (partition_points(*this, indices, first_string, first_binary, end)) { - std::inplace_merge(first_string, first_binary, end, [&](auto a, auto b) { - return get(a) < get(b); - }); - } - if (!ascending) { - std::reverse(indices.begin(), indices.end()); - } -} - } // namespace realm diff --git a/src/realm/set.hpp b/src/realm/set.hpp index fbfa2cbea30..91b83ebbcc7 100644 --- a/src/realm/set.hpp +++ b/src/realm/set.hpp @@ -792,9 +792,6 @@ inline void Set::sort(std::vector& indices, bool ascending) const set_sorted_indices(sz, indices, ascending); } -template <> -void Set::sort(std::vector& indices, bool ascending) const; - template inline void Set::distinct(std::vector& indices, util::Optional sort_order) const { diff --git a/src/realm/sync/CMakeLists.txt b/src/realm/sync/CMakeLists.txt index 0dff70d9432..55ab9ba1370 100644 --- a/src/realm/sync/CMakeLists.txt +++ b/src/realm/sync/CMakeLists.txt @@ -10,7 +10,6 @@ set(SYNC_SOURCES noinst/pending_bootstrap_store.cpp noinst/protocol_codec.cpp noinst/sync_metadata_schema.cpp - binding_callback_thread_observer.cpp changeset_encoder.cpp changeset_parser.cpp changeset.cpp @@ -36,7 +35,6 @@ set(IMPL_INSTALL_HEADERS ) set(SYNC_INSTALL_HEADERS - binding_callback_thread_observer.hpp config.hpp changeset_encoder.hpp changeset_parser.hpp diff --git a/src/realm/sync/changeset.hpp b/src/realm/sync/changeset.hpp index ce4ff4a7eef..e37a29a7f0b 100644 --- a/src/realm/sync/changeset.hpp +++ b/src/realm/sync/changeset.hpp @@ -13,10 +13,7 @@ namespace sync { using InternStrings = std::vector; struct BadChangesetError : ExceptionWithBacktrace { - BadChangesetError(const std::string& msg) - : ExceptionWithBacktrace(util::format("%1. Please contact support", msg)) - { - } + using ExceptionWithBacktrace::ExceptionWithBacktrace; }; struct Changeset { diff --git a/src/realm/sync/client.cpp b/src/realm/sync/client.cpp index eb43ef82b34..c8e162aa3ff 100644 --- a/src/realm/sync/client.cpp +++ b/src/realm/sync/client.cpp @@ -8,7 +8,6 @@ #include "realm/util/optional.hpp" #include #include -#include #include #include #include @@ -215,8 +214,6 @@ class SessionWrapper final : public util::AtomicRefCountBase, public SyncTransac util::Future send_test_command(std::string body); - void handle_pending_client_reset_acknowledgement(); - private: ClientImpl& m_client; DBRef m_db; @@ -393,13 +390,14 @@ SessionWrapperStack::~SessionWrapperStack() ClientImpl::~ClientImpl() { + bool client_destroyed_while_still_running = m_running; + REALM_ASSERT_RELEASE(!client_destroyed_while_still_running); + // Since no other thread is allowed to be accessing this client or any of // its subobjects at this time, no mutex locking is necessary. - drain(); // Session wrappers are removed from m_unactualized_session_wrappers as they // are abandoned. - REALM_ASSERT(m_stopped); REALM_ASSERT(m_unactualized_session_wrappers.empty()); } @@ -444,7 +442,7 @@ bool ClientImpl::wait_for_session_terminations_or_client_stopped() // Thread safety required { - std::lock_guard lock{m_mutex}; + util::LockGuard lock{m_mutex}; m_sessions_terminated = false; } @@ -471,14 +469,14 @@ bool ClientImpl::wait_for_session_terminations_or_client_stopped() else if (!status.is_ok()) throw ExceptionForStatus(status); - std::lock_guard lock{m_mutex}; + util::LockGuard lock{m_mutex}; m_sessions_terminated = true; m_wait_or_client_stopped_cond.notify_all(); }); // Throws bool completion_condition_was_satisfied; { - std::unique_lock lock{m_mutex}; + util::LockGuard lock{m_mutex}; while (!m_sessions_terminated && !m_stopped) m_wait_or_client_stopped_cond.wait(lock); completion_condition_was_satisfied = !m_stopped; @@ -487,44 +485,21 @@ bool ClientImpl::wait_for_session_terminations_or_client_stopped() } -void ClientImpl::drain_connections_on_loop() -{ - post([this](Status status) mutable { - REALM_ASSERT(status.is_ok()); - actualize_and_finalize_session_wrappers(); - drain_connections(); - }); -} - -void ClientImpl::drain() -{ - stop(); - { - std::lock_guard lock{m_drain_mutex}; - if (m_drained) { - return; - } - } - - drain_connections_on_loop(); - - std::unique_lock lock{m_drain_mutex}; - - logger.debug("Waiting for %1 connections to drain", m_num_connections); - m_drain_cv.wait(lock, [&] { - return m_num_connections == 0 && m_outstanding_posts == 0; - }); - - m_drained = true; -} - void ClientImpl::stop() noexcept { - std::lock_guard lock{m_mutex}; + util::LockGuard lock{m_mutex}; if (m_stopped) return; m_stopped = true; m_wait_or_client_stopped_cond.notify_all(); + m_socket_provider->stop(); +} + + +void ClientImpl::run() +{ + auto ta = util::make_temp_assign(m_running, true); + m_socket_provider->run(); } @@ -532,7 +507,7 @@ void ClientImpl::register_unactualized_session_wrapper(SessionWrapper* wrapper, { // Thread safety required. - std::lock_guard lock{m_mutex}; + util::LockGuard lock{m_mutex}; REALM_ASSERT(m_actualize_and_finalize); m_unactualized_session_wrappers.emplace(wrapper, std::move(endpoint)); // Throws bool retrigger = !m_actualize_and_finalize_needed; @@ -555,7 +530,7 @@ void ClientImpl::register_abandoned_session_wrapper(util::bind_ptr unactualized_session_wrappers; SessionWrapperStack abandoned_session_wrappers; { - std::lock_guard lock{m_mutex}; + util::LockGuard lock{m_mutex}; m_actualize_and_finalize_needed = false; swap(m_unactualized_session_wrappers, unactualized_session_wrappers); swap(m_abandoned_session_wrappers, abandoned_session_wrappers); @@ -637,10 +612,6 @@ ClientImpl::get_connection(ServerEndpoint endpoint, const std::string& authoriza } m_prev_connection_ident = ident; was_created = true; - { - std::lock_guard lk(m_drain_mutex); - ++m_num_connections; - } return conn; } @@ -665,12 +636,6 @@ void ClientImpl::remove_connection(ClientImpl::Connection& conn) noexcept REALM_ASSERT(&*j->second == &conn); server_slot.alt_connections.erase(j); } - - { - std::lock_guard lk(m_drain_mutex); - --m_num_connections; - m_drain_cv.notify_all(); - } } @@ -766,11 +731,6 @@ void SessionImpl::on_resumed() m_wrapper.on_resumed(); // Throws } -void SessionImpl::handle_pending_client_reset_acknowledgement() -{ - m_wrapper.handle_pending_client_reset_acknowledgement(); -} - bool SessionImpl::process_flx_bootstrap_message(const SyncProgress& progress, DownloadBatchState batch_state, int64_t query_version, const ReceivedChangesets& received_changesets) @@ -786,18 +746,7 @@ bool SessionImpl::process_flx_bootstrap_message(const SyncProgress& progress, Do } bool new_batch = false; - try { - bootstrap_store->add_batch(query_version, std::move(maybe_progress), received_changesets, &new_batch); - } - catch (const LogicError& ex) { - if (ex.kind() == LogicError::binary_too_big) { - IntegrationException ex(ClientError::bad_changeset_size, - "bootstrap changeset too large to store in pending bootstrap store"); - on_integration_failure(ex); - return true; - } - throw; - } + bootstrap_store->add_batch(query_version, std::move(maybe_progress), received_changesets, &new_batch); // If we've started a new batch and there is more to come, call on_flx_sync_progress to mark the subscription as // bootstrapping. @@ -848,16 +797,11 @@ void SessionImpl::process_pending_flx_bootstrap() int64_t query_version = -1; size_t changesets_processed = 0; - // Used to commit each batch after it was transformed. - TransactionRef transact = get_db()->start_write(); while (bootstrap_store->has_pending()) { auto start_time = std::chrono::steady_clock::now(); auto pending_batch = bootstrap_store->peek_pending(m_wrapper.m_flx_bootstrap_batch_size_bytes); if (!pending_batch.progress) { logger.info("Incomplete pending bootstrap found for query version %1", pending_batch.query_version); - // Close the write transation before clearing the bootstrap store to avoid a deadlock because the - // bootstrap store requires a write transaction itself. - transact->close(); bootstrap_store->clear(); return; } @@ -874,7 +818,6 @@ void SessionImpl::process_pending_flx_bootstrap() history.integrate_server_changesets( *pending_batch.progress, &downloadable_bytes, pending_batch.changesets, new_version, batch_state, logger, - transact, [&](const TransactionRef& tr, util::Span changesets_applied) { REALM_ASSERT_3(changesets_applied.size(), <=, pending_batch.changesets.size()); bootstrap_store->pop_front_pending(tr, changesets_applied.size()); @@ -1337,7 +1280,7 @@ bool SessionWrapper::wait_for_upload_complete_or_client_stopped() std::int_fast64_t target_mark; { - std::lock_guard lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; target_mark = ++m_target_upload_mark; } @@ -1364,7 +1307,7 @@ bool SessionWrapper::wait_for_upload_complete_or_client_stopped() bool completion_condition_was_satisfied; { - std::unique_lock lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; while (m_reached_upload_mark < target_mark && !m_client.m_stopped) m_client.m_wait_or_client_stopped_cond.wait(lock); completion_condition_was_satisfied = !m_client.m_stopped; @@ -1380,7 +1323,7 @@ bool SessionWrapper::wait_for_download_complete_or_client_stopped() std::int_fast64_t target_mark; { - std::lock_guard lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; target_mark = ++m_target_download_mark; } @@ -1407,7 +1350,7 @@ bool SessionWrapper::wait_for_download_complete_or_client_stopped() bool completion_condition_was_satisfied; { - std::unique_lock lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; while (m_reached_download_mark < target_mark && !m_client.m_stopped) m_client.m_wait_or_client_stopped_cond.wait(lock); completion_condition_was_satisfied = !m_client.m_stopped; @@ -1585,7 +1528,7 @@ void SessionWrapper::on_upload_completion() m_download_completion_handlers.push_back(std::move(handler)); // Throws m_sync_completion_handlers.pop_back(); } - std::lock_guard lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; if (m_staged_upload_mark > m_reached_upload_mark) { m_reached_upload_mark = m_staged_upload_mark; m_client.m_wait_or_client_stopped_cond.notify_all(); @@ -1616,7 +1559,7 @@ void SessionWrapper::on_download_completion() m_flx_pending_mark_version = SubscriptionSet::EmptyVersion; } - std::lock_guard lock{m_client.m_mutex}; + util::LockGuard lock{m_client.m_mutex}; if (m_staged_download_mark > m_reached_download_mark) { m_reached_download_mark = m_staged_download_mark; m_client.m_wait_or_client_stopped_cond.notify_all(); @@ -1701,49 +1644,6 @@ util::Future SessionWrapper::send_test_command(std::string body) return m_sess->send_test_command(std::move(body)); } -void SessionWrapper::handle_pending_client_reset_acknowledgement() -{ - auto pending_reset = [&] { - auto ft = m_db->start_frozen(); - return _impl::client_reset::has_pending_reset(ft); - }(); - REALM_ASSERT(pending_reset); - m_sess->logger.info("Tracking pending client reset of type \"%1\" from %2", pending_reset->type, - pending_reset->time); - util::bind_ptr self(this); - async_wait_for(true, true, [self = std::move(self), pending_reset = *pending_reset](std::error_code ec) { - if (ec == util::error::operation_aborted) { - return; - } - auto& logger = self->m_sess->logger; - if (ec) { - logger.error("Error while tracking client reset acknowledgement: %1", ec.message()); - return; - } - - auto wt = self->m_db->start_write(); - auto cur_pending_reset = _impl::client_reset::has_pending_reset(wt); - if (!cur_pending_reset) { - logger.debug( - "Was going to remove client reset tracker for type \"%1\" from %2, but it was already removed", - pending_reset.type, pending_reset.time); - return; - } - else if (cur_pending_reset->type != pending_reset.type || cur_pending_reset->time != pending_reset.time) { - logger.debug( - "Was going to remove client reset tracker for type \"%1\" from %2, but found type \"%3\" from %4.", - pending_reset.type, pending_reset.time, cur_pending_reset->type, cur_pending_reset->time); - } - else { - logger.debug("Client reset of type \"%1\" from %2 has been acknowledged by the server. " - "Removing cycle detection tracker.", - pending_reset.type, pending_reset.time); - } - _impl::client_reset::remove_pending_client_resets(wt); - wt->commit(); - }); -} - // ################ ClientImpl::Connection ################ ClientImpl::Connection::Connection(ClientImpl& client, connection_ident_type ident, ServerEndpoint endpoint, @@ -1871,16 +1771,18 @@ Client::Client(Client&& client) noexcept Client::~Client() noexcept {} -void Client::stop() noexcept +void Client::run() { - m_impl->stop(); + m_impl->run(); // Throws } -void Client::drain() + +void Client::stop() noexcept { - m_impl->drain(); + m_impl->stop(); } + void Client::cancel_reconnect_delay() { m_impl->cancel_reconnect_delay(); diff --git a/src/realm/sync/client.hpp b/src/realm/sync/client.hpp index b743ccf36d9..814557637d4 100644 --- a/src/realm/sync/client.hpp +++ b/src/realm/sync/client.hpp @@ -12,7 +12,6 @@ #include #include #include -#include #include namespace realm::sync { @@ -41,17 +40,13 @@ class Client { /// Run the internal event-loop of the client. At most one thread may /// execute run() at any given time. The call will not return until somebody /// calls stop(). - void run() noexcept; + void run(); /// See run(). /// /// Thread-safe. void stop() noexcept; - /// Forces all connections to close and waits for any pending work on the event - /// loop to complete. All sessions must be destroyed before calling drain. - void drain(); - /// \brief Cancel current or next reconnect delay for all servers. /// /// This corresponds to calling Session::cancel_reconnect_delay() on all diff --git a/src/realm/sync/client_base.hpp b/src/realm/sync/client_base.hpp index a42db0828aa..80980b4fe78 100644 --- a/src/realm/sync/client_base.hpp +++ b/src/realm/sync/client_base.hpp @@ -62,8 +62,9 @@ struct ClientReset { realm::ClientResyncMode mode; DBRef fresh_copy; bool recovery_is_allowed = true; - util::UniqueFunction notify_before_client_reset; - util::UniqueFunction notify_after_client_reset; + util::UniqueFunction notify_before_client_reset; + util::UniqueFunction + notify_after_client_reset; }; /// \brief Protocol errors discovered by the client. @@ -133,6 +134,46 @@ static constexpr milliseconds_type default_fast_reconnect_limit = 60000; // 1 using RoundtripTimeHandler = void(milliseconds_type roundtrip_time); struct ClientConfig { + /// + /// DEPRECATED - Will be removed in a future release + /// + /// An optional custom platform description to be sent to server as part + /// of a user agent description (HTTP `User-Agent` header). + /// + /// If left empty, the platform description will be whatever is returned + /// by util::get_platform_info(). + std::string user_agent_platform_info; + + /// + /// DEPRECATED - Will be removed in a future release + /// + /// Optional information about the application to be added to the user + /// agent description as sent to the server. The intention is that the + /// application describes itself using the following (rough) syntax: + /// + /// ::= ( )* + /// ::= "/" [
] + /// ::= ()+ + /// ::= ( | "." | "-" | "_")* + ///
::= + /// ::= "(" ( | )* ")" + /// + /// Where `` is a single space character, `` is a decimal + /// digit, `` is any alphanumeric character, and `` is + /// any character other than `(` and `)`. + /// + /// When multiple levels are present, the innermost layer (the one that + /// is closest to this API) should appear first. + /// + /// Example: + /// + /// RealmJS/2.13.0 RealmStudio/2.9.0 + /// + /// Note: The user agent description is not intended for machine + /// interpretation, but should still follow the specified syntax such + /// that it remains easily interpretable by human beings. + std::string user_agent_application_info; + /// An optional logger to be used by the client. If no logger is /// specified, the client will use an instance of util::StderrLogger /// with the log level threshold set to util::Logger::Level::info. The diff --git a/src/realm/sync/network/default_socket.cpp b/src/realm/sync/network/default_socket.cpp index 0ea4f83d786..f66aae84aaa 100644 --- a/src/realm/sync/network/default_socket.cpp +++ b/src/realm/sync/network/default_socket.cpp @@ -1,30 +1,23 @@ #include -#include - #include #include #include -#include namespace realm::sync::websocket { namespace { - -/// -/// DefaultWebSocketImpl - websocket implementation for the default socket provider -/// -class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { +class DefaultWebSocketImpl final : public WebSocketInterface, public Config { public: DefaultWebSocketImpl(const std::shared_ptr& logger_ptr, network::Service& service, - std::mt19937_64& random, const std::string user_agent, - std::unique_ptr observer, WebSocketEndpoint&& endpoint) + std::mt19937_64& random, const std::string user_agent, WebSocketObserver& observer, + WebSocketEndpoint&& endpoint) : m_logger_ptr{logger_ptr} , m_logger{*m_logger_ptr} , m_random{random} , m_service{service} , m_user_agent{user_agent} - , m_observer{std::move(observer)} + , m_observer{observer} , m_endpoint{std::move(endpoint)} , m_websocket(*this) { @@ -43,11 +36,6 @@ class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { return m_app_services_coid; } - void force_handshake_response_for_testing(int status_code, std::string body = "") override - { - m_websocket.force_handshake_response_for_testing(status_code, body); - } - // public for HTTPClient CRTP, but not on the EZSocket interface, so de-facto private void async_read(char*, std::size_t, ReadCompletionHandler) override; void async_read_until(char*, std::size_t, char, ReadCompletionHandler) override; @@ -56,9 +44,9 @@ class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { private: using milliseconds_type = std::int_fast64_t; - const std::shared_ptr& websocket_get_logger() noexcept override + util::Logger& websocket_get_logger() noexcept override { - return m_logger_ptr; + return m_logger; } std::mt19937_64& websocket_get_random() noexcept override { @@ -72,104 +60,34 @@ class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { m_app_services_coid = it->second; } auto it = headers.find("Sec-WebSocket-Protocol"); - m_observer->websocket_connected_handler(it == headers.end() ? empty : it->second); + m_observer.websocket_connected_handler(it == headers.end() ? empty : it->second); } void websocket_read_error_handler(std::error_code ec) override { m_logger.error("Reading failed: %1", ec.message()); // Throws - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, Status{ErrorCodes::ReadError, ec.message()}); + m_observer.websocket_read_or_write_error_handler(ec); } void websocket_write_error_handler(std::error_code ec) override { m_logger.error("Writing failed: %1", ec.message()); // Throws - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, Status{ErrorCodes::WriteError, ec.message()}); + m_observer.websocket_read_or_write_error_handler(ec); } void websocket_handshake_error_handler(std::error_code ec, const HTTPHeaders*, const std::string_view* body) override { - ErrorCodes::Error error; - bool was_clean = true; - - if (ec == websocket::Error::bad_response_301_moved_permanently || - ec == websocket::Error::bad_response_308_permanent_redirect) { - error = ErrorCodes::WebSocket_MovedPermanently; - } - else if (ec == websocket::Error::bad_response_3xx_redirection) { - error = ErrorCodes::WebSocket_Retry_Error; - was_clean = false; - } - else if (ec == websocket::Error::bad_response_401_unauthorized) { - error = ErrorCodes::WebSocket_Unauthorized; - } - else if (ec == websocket::Error::bad_response_403_forbidden) { - error = ErrorCodes::WebSocket_Forbidden; - } - else if (ec == websocket::Error::bad_response_5xx_server_error || - ec == websocket::Error::bad_response_500_internal_server_error || - ec == websocket::Error::bad_response_502_bad_gateway || - ec == websocket::Error::bad_response_503_service_unavailable || - ec == websocket::Error::bad_response_504_gateway_timeout) { - error = ErrorCodes::WebSocket_InternalServerError; - was_clean = false; - } - else { - error = ErrorCodes::WebSocket_Fatal_Error; - was_clean = false; - if (body) { - std::string_view identifier = "REALM_SYNC_PROTOCOL_MISMATCH"; - auto i = body->find(identifier); - if (i != std::string_view::npos) { - std::string_view rest = body->substr(i + identifier.size()); - // FIXME: Use std::string_view::begins_with() in C++20. - auto begins_with = [](std::string_view string, std::string_view prefix) { - return (string.size() >= prefix.size() && - std::equal(string.data(), string.data() + prefix.size(), prefix.data())); - }; - if (begins_with(rest, ":CLIENT_TOO_OLD")) { - error = ErrorCodes::WebSocket_Client_Too_Old; - } - else if (begins_with(rest, ":CLIENT_TOO_NEW")) { - error = ErrorCodes::WebSocket_Client_Too_New; - } - else { - // Other more complicated forms of mismatch - error = ErrorCodes::WebSocket_Protocol_Mismatch; - } - was_clean = true; - } - } - } - - websocket_error_and_close_handler(was_clean, Status{error, ec.message()}); + m_observer.websocket_handshake_error_handler(ec, body); } void websocket_protocol_error_handler(std::error_code ec) override { - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, Status{ErrorCodes::WebSocket_ProtocolError, ec.message()}); + m_observer.websocket_protocol_error_handler(ec); } bool websocket_close_message_received(std::error_code ec, StringData message) override { - constexpr bool was_clean = true; - - // Normal closure. - if (ec.value() == 1000) { - return websocket_error_and_close_handler(was_clean, Status::OK()); - } - return websocket_error_and_close_handler(was_clean, - Status{static_cast(ec.value()), message}); - } - bool websocket_error_and_close_handler(bool was_clean, Status status) - { - if (!was_clean) { - m_observer->websocket_error_handler(); - } - return m_observer->websocket_closed_handler(was_clean, status); + return m_observer.websocket_close_message_received(ec, message); } bool websocket_binary_message_received(const char* ptr, std::size_t size) override { - return m_observer->websocket_binary_message_received(util::Span(ptr, size)); + return m_observer.websocket_binary_message_received(util::Span(ptr, size)); } void initiate_resolve(); @@ -177,6 +95,7 @@ class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { void initiate_tcp_connect(network::Endpoint::List, std::size_t); void handle_tcp_connect(std::error_code, network::Endpoint::List, std::size_t); void initiate_http_tunnel(); + void handle_http_tunnel(std::error_code); void initiate_websocket_or_ssl_handshake(); void initiate_ssl_handshake(); void handle_ssl_handshake(std::error_code); @@ -196,7 +115,7 @@ class DefaultWebSocketImpl final : public DefaultWebSocket, public Config { const std::string m_user_agent; std::string m_app_services_coid; - std::unique_ptr m_observer; + WebSocketObserver& m_observer; const WebSocketEndpoint m_endpoint; util::Optional m_resolver; @@ -272,8 +191,7 @@ void DefaultWebSocketImpl::handle_resolve(std::error_code ec, network::Endpoint: { if (ec) { m_logger.error("Failed to resolve '%1:%2': %3", m_endpoint.address, m_endpoint.port, ec.message()); // Throws - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, Status{ErrorCodes::ResolveFailed, ec.message()}); // Throws + m_observer.websocket_connect_error_handler(ec); // Throws return; } @@ -312,8 +230,7 @@ void DefaultWebSocketImpl::handle_tcp_connect(std::error_code ec, network::Endpo } // All endpoints failed m_logger.error("Failed to connect to '%1:%2': All endpoints failed", m_endpoint.address, m_endpoint.port); - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, Status{ErrorCodes::ConnectionFailed, ec.message()}); // Throws + m_observer.websocket_connect_error_handler(ec); // Throws return; } @@ -348,21 +265,19 @@ void DefaultWebSocketImpl::initiate_http_tunnel() req.headers.emplace("Host", util::format("%1:%2", m_endpoint.address, m_endpoint.port)); // TODO handle proxy authorization - m_proxy_client.emplace(*this, m_logger_ptr); + m_proxy_client.emplace(*this, m_logger); auto handler = [this](HTTPResponse response, std::error_code ec) { if (ec && ec != util::error::operation_aborted) { m_logger.error("Failed to establish HTTP tunnel: %1", ec.message()); - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, - Status{ErrorCodes::ConnectionFailed, ec.message()}); // Throws + m_observer.websocket_connect_error_handler(ec); // Throws return; } if (response.status != HTTPStatus::Ok) { m_logger.error("Proxy server returned response '%1 %2'", response.status, response.reason); // Throws - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, - Status{ErrorCodes::ConnectionFailed, response.reason}); // Throws + std::error_code ec2 = + websocket::Error::bad_response_unexpected_status_code; // FIXME: is this the right error? + m_observer.websocket_connect_error_handler(ec2); // Throws return; } @@ -424,9 +339,7 @@ void DefaultWebSocketImpl::handle_ssl_handshake(std::error_code ec) { if (ec) { REALM_ASSERT(ec != util::error::operation_aborted); - constexpr bool was_clean = false; - websocket_error_and_close_handler(was_clean, - Status{ErrorCodes::WebSocket_TLSHandshakeFailed, ec.message()}); // Throws + m_observer.websocket_ssl_handshake_error_handler(ec); // Throws return; } @@ -458,175 +371,11 @@ void DefaultWebSocketImpl::initiate_websocket_handshake() } } // namespace -/// -/// DefaultSocketProvider - default socket provider implementation -/// - -DefaultSocketProvider::DefaultSocketProvider(const std::shared_ptr& logger, - const std::string user_agent, AutoStart auto_start) - : m_logger_ptr{logger} - , m_service{} - , m_random{} - , m_user_agent{user_agent} - , m_mutex{} - , m_state{State::Stopped} - , m_state_cv{} - , m_thread{} -{ - REALM_ASSERT(m_logger_ptr); // Make sure the logger is valid - util::seed_prng_nondeterministically(m_random); // Throws - if (auto_start) { - start(); - } -} - -DefaultSocketProvider::~DefaultSocketProvider() -{ - m_logger_ptr->trace("Default event loop teardown"); - // Wait for the thread to stop - stop(true); - // Shutting down - no need to lock mutex before check - REALM_ASSERT(m_state == State::Stopped); -} - -void DefaultSocketProvider::start() -{ - std::unique_lock lock(m_mutex); - // Has the thread already been started or is running - if (m_state == State::Starting || m_state == State::Running) - return; // early return - - // If the thread has been previously run, make sure it has been joined first - if (m_state == State::Stopping) { - state_wait_for(lock, State::Stopped); - } - - m_logger_ptr->trace("Default event loop: start()"); - REALM_ASSERT(m_state == State::Stopped); - do_state_update(lock, State::Starting); - m_thread = std::thread{&DefaultSocketProvider::event_loop, this}; - // Wait for the thread to start before continuing - state_wait_for(lock, State::Running); -} - -void DefaultSocketProvider::event_loop() -{ - m_logger_ptr->trace("Default event loop: thread running"); - auto will_destroy_thread = util::make_scope_exit([&]() noexcept { - m_logger_ptr->trace("Default event loop: thread exiting"); - if (g_binding_callback_thread_observer) - g_binding_callback_thread_observer->will_destroy_thread(); - - std::unique_lock lock(m_mutex); - // Did we get here due to an unhandled exception? - if (m_state != State::Stopping) { - m_logger_ptr->error("Default event loop: thread exited unexpectedly"); - } - m_state = State::Stopped; - std::notify_all_at_thread_exit(m_state_cv, std::move(lock)); - }); - - if (g_binding_callback_thread_observer) - g_binding_callback_thread_observer->did_create_thread(); - - { - std::lock_guard lock(m_mutex); - REALM_ASSERT(m_state == State::Starting); - } - - // We update the state to Running from inside the event loop so that start() is blocked until - // the event loop is actually ready to receive work. - m_service.post([this, my_generation = ++m_event_loop_generation](Status status) { - if (status == ErrorCodes::OperationAborted) { - return; - } - - REALM_ASSERT(status.is_ok()); - - std::unique_lock lock(m_mutex); - // This is a callback from a previous generation - if (m_event_loop_generation != my_generation) { - return; - } - if (m_state == State::Stopping) { - return; - } - m_logger_ptr->trace("Default event loop: service run"); - REALM_ASSERT(m_state == State::Starting); - do_state_update(lock, State::Running); - }); - - try { - m_service.run_until_stopped(); // Throws - } - catch (const std::exception& e) { - std::unique_lock lock(m_mutex); - // Service is no longer running, event loop thread is stopping - do_state_update(lock, State::Stopping); - lock.unlock(); - m_logger_ptr->error("Default event loop exception: ", e.what()); - if (g_binding_callback_thread_observer) - g_binding_callback_thread_observer->handle_error(e); - else - throw; - } -} - -void DefaultSocketProvider::stop(bool wait_for_stop) -{ - std::unique_lock lock(m_mutex); - - // Do nothing if the thread is not started or running or stop has already been called - if (m_state == State::Starting || m_state == State::Running) { - m_logger_ptr->trace("Default event loop: stop()"); - do_state_update(lock, State::Stopping); - // Updating state to Stopping will free a start() if it is waiting for the thread to - // start and may cause the thread to exit early before calling service.run() - m_service.stop(); // Unblocks m_service.run() - } - - // Wait until the thread is stopped (exited) if requested - if (wait_for_stop) { - m_logger_ptr->trace("Default event loop: wait for stop"); - state_wait_for(lock, State::Stopped); - if (m_thread.joinable()) { - m_thread.join(); - } - } -} - -// +---------------------------------------+ -// \/ | -// State Machine: Stopped -> Starting -> Running -> Stopping -+ -// | | ^ -// +----------------------+ - -void DefaultSocketProvider::do_state_update(std::unique_lock&, State new_state) -{ - // m_state_mutex should already be locked... - m_state = new_state; - m_state_cv.notify_all(); // Let any waiters check the state -} - -void DefaultSocketProvider::state_wait_for(std::unique_lock& lock, State expected_state) -{ - // Check for condition already met or superseded - if (m_state >= expected_state) - return; - - m_state_cv.wait(lock, [this, expected_state]() { - // are we there yet? - if (m_state < expected_state) - return false; - return true; - }); -} - -std::unique_ptr DefaultSocketProvider::connect(std::unique_ptr observer, +std::unique_ptr DefaultSocketProvider::connect(WebSocketObserver* observer, WebSocketEndpoint&& endpoint) { - return std::make_unique(m_logger_ptr, m_service, m_random, m_user_agent, - std::move(observer), std::move(endpoint)); + return std::make_unique(m_logger_ptr, *m_service, m_random, m_user_agent, *observer, + std::move(endpoint)); } } // namespace realm::sync::websocket diff --git a/src/realm/sync/network/default_socket.hpp b/src/realm/sync/network/default_socket.hpp index a5a7c30f6f0..87f5159d540 100644 --- a/src/realm/sync/network/default_socket.hpp +++ b/src/realm/sync/network/default_socket.hpp @@ -8,9 +8,7 @@ #include #include #include -#include #include -#include namespace realm::sync::network { class Service; @@ -45,71 +43,64 @@ class DefaultSocketProvider : public SyncSocketProvider { network::DeadlineTimer m_timer; }; - struct AutoStartTag { - }; - - using AutoStart = util::TaggedBool; - DefaultSocketProvider(const std::shared_ptr& logger, const std::string user_agent, - AutoStart auto_start = AutoStart{true}); + DefaultSocketProvider(const std::shared_ptr& logger, const std::string user_agent) + : m_logger_ptr{logger} + , m_service{std::make_shared()} + , m_random{} + , m_user_agent{user_agent} + { + REALM_ASSERT(m_logger_ptr); // Make sure the logger is valid + REALM_ASSERT(m_service); // Make sure the service is valid + util::seed_prng_nondeterministically(m_random); // Throws + start_keep_running_timer(); + } // Don't allow move or copy constructor DefaultSocketProvider(DefaultSocketProvider&&) = delete; - ~DefaultSocketProvider(); - - // Start the event loop if it is not started already. Otherwise, do nothing. - void start(); + // Temporary workaround until event loop is completely moved here + void run() override + { + m_service->run(); + } - /// Temporary workaround until client shutdown has been updated in a separate PR - these functions - /// will be handled internally when this happens. - /// Stops the internal event loop (provided by network::Service) - void stop(bool wait_for_stop = false) override; + void stop() override + { + m_service->stop(); + } - std::unique_ptr connect(std::unique_ptr, WebSocketEndpoint&&) override; + std::unique_ptr connect(WebSocketObserver*, WebSocketEndpoint&&) override; void post(FunctionHandler&& handler) override { + REALM_ASSERT(m_service); // Don't post empty handlers onto the event loop if (!handler) return; - m_service.post(std::move(handler)); + m_service->post(std::move(handler)); } SyncTimer create_timer(std::chrono::milliseconds delay, FunctionHandler&& handler) override { - return std::unique_ptr(new DefaultSocketProvider::Timer(m_service, delay, std::move(handler))); + return std::unique_ptr(new DefaultSocketProvider::Timer(*m_service, delay, std::move(handler))); } private: - enum class State { Starting, Running, Stopping, Stopped }; - - /// Block until the state reaches the expected or later state - return true if state matches expected state - void state_wait_for(std::unique_lock& lock, State expected_state); - /// Internal function for updating the state and signaling the wait_for_state condvar - void do_state_update(std::unique_lock&, State new_state); - /// The execution code for the event loop thread - void event_loop(); + // TODO: Revisit Service::run() so the keep running timer is no longer needed + void start_keep_running_timer() + { + auto handler = [this](Status status) { + if (status.code() != ErrorCodes::OperationAborted) + start_keep_running_timer(); + }; + m_keep_running_timer = create_timer(std::chrono::hours(1000), std::move(handler)); // Throws + } std::shared_ptr m_logger_ptr; - network::Service m_service; + std::shared_ptr m_service; std::mt19937_64 m_random; const std::string m_user_agent; - std::mutex m_mutex; - uint64_t m_event_loop_generation = 0; - State m_state; // protected by m_mutex - std::condition_variable m_state_cv; // uses m_mutex - std::thread m_thread; // protected by m_mutex -}; - -/// Class for the Default Socket Provider websockets that allows a simulated -/// http response to be specified for testing. -class DefaultWebSocket : public WebSocketInterface { -public: - virtual ~DefaultWebSocket() = default; - - virtual void force_handshake_response_for_testing(int status_code, std::string body = "") = 0; - -protected: + SyncTimer m_keep_running_timer; }; } // namespace realm::sync::websocket diff --git a/src/realm/sync/network/http.hpp b/src/realm/sync/network/http.hpp index 06a0ef59452..acc4e789bc3 100644 --- a/src/realm/sync/network/http.hpp +++ b/src/realm/sync/network/http.hpp @@ -194,7 +194,8 @@ std::ostream& operator<<(std::ostream&, HTTPStatus); struct HTTPParserBase { - const std::shared_ptr logger_ptr; + // An HTTPParserBase is tied to to an HTTPClient or HTTPServer, which are owned + // by either a Websocket or ServerImpl class, so no need for a shared_ptr util::Logger& logger; // FIXME: Generally useful? @@ -205,9 +206,8 @@ struct HTTPParserBase { } }; - HTTPParserBase(const std::shared_ptr& logger_ptr) - : logger_ptr{logger_ptr} - , logger{*logger_ptr} + HTTPParserBase(util::Logger& logger) + : logger{logger} { // Allocating read buffer with calloc to avoid accidentally spilling // data from other sessions in case of a buffer overflow exploit. @@ -255,8 +255,8 @@ struct HTTPParserBase { template struct HTTPParser : protected HTTPParserBase { - explicit HTTPParser(Socket& socket, const std::shared_ptr& logger_ptr) - : HTTPParserBase(logger_ptr) + explicit HTTPParser(Socket& socket, util::Logger& logger) + : HTTPParserBase(logger) , m_socket(socket) { } @@ -346,8 +346,8 @@ template struct HTTPClient : protected HTTPParser { using Handler = void(HTTPResponse, std::error_code); - explicit HTTPClient(Socket& socket, const std::shared_ptr& logger_ptr) - : HTTPParser(socket, logger_ptr) + explicit HTTPClient(Socket& socket, util::Logger& logger) + : HTTPParser(socket, logger) { } @@ -429,8 +429,8 @@ struct HTTPServer : protected HTTPParser { using RequestHandler = void(HTTPRequest, std::error_code); using RespondHandler = void(std::error_code); - explicit HTTPServer(Socket& socket, const std::shared_ptr& logger_ptr) - : HTTPParser(socket, logger_ptr) + explicit HTTPServer(Socket& socket, util::Logger& logger) + : HTTPParser(socket, logger) { } diff --git a/src/realm/sync/network/network.cpp b/src/realm/sync/network/network.cpp index b0efee866f6..6df7fb8ddd2 100644 --- a/src/realm/sync/network/network.cpp +++ b/src/realm/sync/network/network.cpp @@ -1,14 +1,12 @@ #define _WINSOCK_DEPRECATED_NO_WARNINGS -#include #include -#include #include -#include +#include +#include #include #include -#include #include @@ -22,6 +20,7 @@ #include #include #include +#include #include #include @@ -318,7 +317,7 @@ class WakeupPipe { // Thread-safe. void signal() noexcept { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; if (!m_signaled) { char c = 0; ssize_t ret = ::write(m_write_fd, &c, 1); @@ -332,7 +331,7 @@ class WakeupPipe { // Thread-safe. void acknowledge_signal() noexcept { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; if (m_signaled) { char c; ssize_t ret = ::read(m_read_fd, &c, 1); @@ -343,7 +342,7 @@ class WakeupPipe { private: CloseGuard m_read_fd, m_write_fd; - std::mutex m_mutex; + Mutex m_mutex; bool m_signaled = false; // Protected by `m_mutex`. }; @@ -1338,7 +1337,7 @@ class Service::Impl { bool resolver_thread_started = m_resolver_thread.joinable(); if (resolver_thread_started) { { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; m_stop_resolver_thread = true; m_resolver_cond.notify_all(); } @@ -1377,18 +1376,74 @@ class Service::Impl { void run() { - run_impl(true); + bool no_incomplete_resolve_operations; + + on_handlers_executed_or_interrupted : { + LockGuard lock{m_mutex}; + if (m_stopped) + return; + // Note: Order of post operations must be preserved. + m_completed_operations.push_back(m_completed_operations_2); + no_incomplete_resolve_operations = (!m_resolve_in_progress && m_resolve_operations.empty()); + + if (m_completed_operations.empty()) + goto on_time_progressed; } - void run_until_stopped() - { - run_impl(false); + on_operations_completed : { +#ifdef REALM_UTIL_NETWORK_EVENT_LOOP_METRICS + m_handler_exec_start_time = clock::now(); +#endif + while (LendersOperPtr op = m_completed_operations.pop_front()) + execute(op); // Throws +#ifdef REALM_UTIL_NETWORK_EVENT_LOOP_METRICS + m_handler_exec_time += clock::now() - m_handler_exec_start_time; +#endif + goto on_handlers_executed_or_interrupted; + } + + on_time_progressed : { + clock::time_point now = clock::now(); + if (process_timers(now)) + goto on_operations_completed; + + bool no_incomplete_operations = + (io_reactor.empty() && m_wait_operations.empty() && no_incomplete_resolve_operations); + if (no_incomplete_operations) { + // We can only get to this point when there are no completion + // handlers ready to execute. It happens either because of a + // fall-through from on_operations_completed, or because of a + // jump to on_time_progressed, but that only happens if no + // completions handlers became ready during + // wait_and_process_io(). + // + // We can also only get to this point when there are no + // asynchronous operations in progress (due to the preceeding + // if-condition. + // + // It is possible that an other thread has added new post + // operations since we checked, but there is really no point in + // rechecking that, as it is always possible, even after a + // recheck, that new post handlers get added after we decide to + // return, but before we actually do return. Also, if would + // offer no additional guarantees to the application. + return; // Out of work + } + + // Blocking wait for I/O + bool interrupted = false; + if (wait_and_process_io(now, interrupted)) // Throws + goto on_operations_completed; + if (interrupted) + goto on_handlers_executed_or_interrupted; + goto on_time_progressed; + } } void stop() noexcept { { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; if (m_stopped) return; m_stopped = true; @@ -1398,7 +1453,7 @@ class Service::Impl { void reset() noexcept { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; m_stopped = false; } @@ -1407,7 +1462,7 @@ class Service::Impl { void add_resolve_oper(LendersResolveOperPtr op) { { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; m_resolve_operations.push_back(std::move(op)); // Throws m_resolver_cond.notify_all(); } @@ -1428,7 +1483,7 @@ class Service::Impl { void post(PostOperConstr constr, std::size_t size, void* cookie) { { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; std::unique_ptr mem; if (m_post_oper && m_post_oper->m_size >= size) { // Reuse old memory @@ -1458,7 +1513,7 @@ class Service::Impl { // Keep the larger memory chunk (`op_2` or m_post_oper) { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; if (!m_post_oper || m_post_oper->m_size < size) swap(op_2, m_post_oper); } @@ -1467,7 +1522,7 @@ class Service::Impl { void trigger_exec(TriggerExecOperBase& op) noexcept { { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; if (op.m_in_use) return; op.m_in_use = true; @@ -1480,7 +1535,7 @@ class Service::Impl { void reset_trigger_exec(TriggerExecOperBase& op) noexcept { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; op.m_in_use = false; } @@ -1496,7 +1551,7 @@ class Service::Impl { void cancel_resolve_oper(ResolveOperBase& op) noexcept { - std::lock_guard lock{m_mutex}; + LockGuard lock{m_mutex}; op.cancel(); } @@ -1533,14 +1588,14 @@ class Service::Impl { using WaitQueue = util::PriorityQueue, WaitOperCompare>; WaitQueue m_wait_operations; - std::mutex m_mutex; + Mutex m_mutex; OwnersOperPtr m_post_oper; // Protected by `m_mutex` OperQueue m_resolve_operations; // Protected by `m_mutex` OperQueue m_completed_operations_2; // Protected by `m_mutex` bool m_stopped = false; // Protected by `m_mutex` bool m_stop_resolver_thread = false; // Protected by `m_mutex` bool m_resolve_in_progress = false; // Protected by `m_mutex` - std::condition_variable m_resolver_cond; // Protected by `m_mutex` + CondVar m_resolver_cond; // Protected by `m_mutex` std::thread m_resolver_thread; @@ -1550,71 +1605,7 @@ class Service::Impl { clock::time_point m_handler_exec_start_time; clock::duration m_handler_exec_time = clock::duration::zero(); #endif - void run_impl(bool return_when_idle) - { - bool no_incomplete_resolve_operations; - - on_handlers_executed_or_interrupted : { - std::lock_guard lock{m_mutex}; - if (m_stopped) - return; - // Note: Order of post operations must be preserved. - m_completed_operations.push_back(m_completed_operations_2); - no_incomplete_resolve_operations = (!m_resolve_in_progress && m_resolve_operations.empty()); - - if (m_completed_operations.empty()) - goto on_time_progressed; - } - - on_operations_completed : { -#ifdef REALM_UTIL_NETWORK_EVENT_LOOP_METRICS - m_handler_exec_start_time = clock::now(); -#endif - while (LendersOperPtr op = m_completed_operations.pop_front()) - execute(op); // Throws -#ifdef REALM_UTIL_NETWORK_EVENT_LOOP_METRICS - m_handler_exec_time += clock::now() - m_handler_exec_start_time; -#endif - goto on_handlers_executed_or_interrupted; - } - - on_time_progressed : { - clock::time_point now = clock::now(); - if (process_timers(now)) - goto on_operations_completed; - - bool no_incomplete_operations = - (io_reactor.empty() && m_wait_operations.empty() && no_incomplete_resolve_operations); - if (no_incomplete_operations && return_when_idle) { - // We can only get to this point when there are no completion - // handlers ready to execute. It happens either because of a - // fall-through from on_operations_completed, or because of a - // jump to on_time_progressed, but that only happens if no - // completions handlers became ready during - // wait_and_process_io(). - // - // We can also only get to this point when there are no - // asynchronous operations in progress (due to the preceeding - // if-condition. - // - // It is possible that an other thread has added new post - // operations since we checked, but there is really no point in - // rechecking that, as it is always possible, even after a - // recheck, that new post handlers get added after we decide to - // return, but before we actually do return. Also, if would - // offer no additional guarantees to the application. - return; // Out of work - } - // Blocking wait for I/O - bool interrupted = false; - if (wait_and_process_io(now, interrupted)) // Throws - goto on_operations_completed; - if (interrupted) - goto on_handlers_executed_or_interrupted; - goto on_time_progressed; - } - } bool process_timers(clock::time_point now) { bool any_operations_completed = false; @@ -1651,7 +1642,7 @@ class Service::Impl { LendersResolveOperPtr op; for (;;) { { - std::unique_lock lock{m_mutex}; + LockGuard lock{m_mutex}; if (op) { m_completed_operations_2.push_back(std::move(op)); io_reactor.interrupt(); @@ -1771,12 +1762,6 @@ void Service::run() } -void Service::run_until_stopped() -{ - m_impl->run_until_stopped(); -} - - void Service::stop() noexcept { m_impl->stop(); diff --git a/src/realm/sync/network/network.hpp b/src/realm/sync/network/network.hpp index b1f55d3f95d..7e40809f84a 100644 --- a/src/realm/sync/network/network.hpp +++ b/src/realm/sync/network/network.hpp @@ -281,10 +281,6 @@ class Service { /// ready. If there are no completion handlers ready for execution, and /// there are no asynchronous operations in progress, run() returns. /// - /// run_until_stopped() will continue running even if there are no completion - /// handlers ready for execution, and no asynchronous operations in progress, - /// until stop() is called. - /// /// All completion handlers, including handlers submitted via post() will be /// executed from run(), that is, by the thread that executes run(). If no /// thread executes run(), then the completion handlers will not be @@ -296,7 +292,6 @@ class Service { /// Syncronous operations (e.g., Socket::connect()) execute independently of /// the event loop, and do not require that any thread calls run(). void run(); - void run_until_stopped(); /// @{ \brief Stop event loop execution. /// @@ -1386,19 +1381,19 @@ enum class ResolveErrors { host_not_found = 1, /// Host not found (non-authoritative). - host_not_found_try_again = 2, + host_not_found_try_again, /// The query is valid but does not have associated address data. - no_data = 3, + no_data, /// A non-recoverable error occurred. - no_recovery = 4, + no_recovery, /// The service is not supported for the given socket type. - service_not_found = 5, + service_not_found, /// The socket type is not supported. - socket_type_not_supported = 6, + socket_type_not_supported }; /// The error category associated with ResolveErrors. The name of this category is diff --git a/src/realm/sync/network/websocket.cpp b/src/realm/sync/network/websocket.cpp index c7f93af9381..4da355f2f63 100644 --- a/src/realm/sync/network/websocket.cpp +++ b/src/realm/sync/network/websocket.cpp @@ -568,9 +568,8 @@ class WebSocket { public: WebSocket(websocket::Config& config) : m_config(config) - , m_logger_ptr(config.websocket_get_logger()) - , m_logger{*m_logger_ptr} - , m_frame_reader(m_logger, m_is_client) + , m_logger(config.websocket_get_logger()) + , m_frame_reader(config.websocket_get_logger(), m_is_client) { m_logger.debug("WebSocket::Websocket()"); } @@ -585,7 +584,7 @@ class WebSocket { m_sec_websocket_key = make_random_sec_websocket_key(m_config.websocket_get_random()); - m_http_client.reset(new HTTPClient(m_config, m_logger_ptr)); + m_http_client.reset(new HTTPClient(m_config, m_logger)); m_frame_reader.reset(); HTTPRequest req; req.method = HTTPMethod::Get; @@ -638,7 +637,7 @@ class WebSocket { m_stopped = false; m_is_client = false; - m_http_server.reset(new HTTPServer(m_config, m_logger_ptr)); + m_http_server.reset(new HTTPServer(m_config, m_logger)); m_frame_reader.reset(); auto handler = [this](HTTPRequest request, std::error_code ec) { @@ -724,15 +723,9 @@ class WebSocket { m_frame_reader.reset(); } - void force_handshake_response_for_testing(int status_code, std::string body) - { - m_test_handshake_response.emplace(status_code); - m_test_handshake_response_body = body; - } - private: websocket::Config& m_config; - const std::shared_ptr m_logger_ptr; + // websocket is owned by the server or websocket factory, so a shared_ptr isn't needed util::Logger& m_logger; FrameReader m_frame_reader; @@ -751,9 +744,6 @@ class WebSocket { util::UniqueFunction m_write_completion_handler; - std::optional m_test_handshake_response; - std::string m_test_handshake_response_body; - void error_client_malformed_response() { m_stopped = true; @@ -773,17 +763,12 @@ class WebSocket { int status_code = int(response.status); std::error_code ec; - if (m_test_handshake_response) - status_code = *m_test_handshake_response; - if (status_code == 200) ec = Error::bad_response_200_ok; else if (status_code >= 200 && status_code < 300) ec = Error::bad_response_2xx_successful; else if (status_code == 301) ec = Error::bad_response_301_moved_permanently; - else if (status_code == 308) - ec = Error::bad_response_308_permanent_redirect; else if (status_code >= 300 && status_code < 400) ec = Error::bad_response_3xx_redirection; else if (status_code == 401) @@ -811,11 +796,7 @@ class WebSocket { std::string_view body; std::string_view* body_ptr = nullptr; - if (m_test_handshake_response) { - body = m_test_handshake_response_body; - body_ptr = &body; - } - else if (response.body) { + if (response.body) { body = *response.body; body_ptr = &body; } @@ -869,8 +850,7 @@ class WebSocket { m_logger.debug("WebSocket::handle_http_response_received()"); m_logger.trace("HTTP response = %1", response); - if (response.status != HTTPStatus::SwitchingProtocols || - (m_test_handshake_response && *m_test_handshake_response != 101)) { + if (response.status != HTTPStatus::SwitchingProtocols) { error_client_response_not_101(response); return; } @@ -1066,8 +1046,6 @@ const char* get_error_message(Error error_code) return "Bad WebSocket response 3xx redirection"; case Error::bad_response_301_moved_permanently: return "Bad WebSocket response 301 moved permanently"; - case Error::bad_response_308_permanent_redirect: - return "Bad WebSocket response 308 permanent redirect"; case Error::bad_response_4xx_client_errors: return "Bad WebSocket response 4xx client errors"; case Error::bad_response_401_unauthorized: @@ -1126,10 +1104,36 @@ class CloseStatusErrorCategory : public std::error_category { { // Converts an error_code to one of the pre-defined status codes in // https://tools.ietf.org/html/rfc6455#section-7.4.1 - if (error_code == 1000 || error_code == 0) { - return ErrorCodes::error_string(ErrorCodes::OK); - } - return ErrorCodes::error_string(static_cast(error_code)); + switch (error_code) { + case 1000: + return "normal closure"; + case 1001: + return "endpoint going away"; + case 1002: + return "protocol error"; + case 1003: + return "invalid data type"; + case 1004: + return "reserved"; + case 1005: + return "no status code present"; + case 1006: + return "no close control frame sent"; + case 1007: + return "message data type mis-match"; + case 1008: + return "policy violation"; + case 1009: + return "message too big"; + case 1010: + return "missing extension"; + case 1011: + return "unexpected error"; + case 1015: + return "TLS handshake failure"; + default: + return "unknown error"; + }; } }; @@ -1234,11 +1238,6 @@ void websocket::Socket::stop() noexcept m_impl->stop(); } -void websocket::Socket::force_handshake_response_for_testing(int status_code, std::string body) -{ - m_impl->force_handshake_response_for_testing(status_code, body); -} - util::Optional websocket::read_sec_websocket_protocol(const HTTPRequest& request) { const HTTPHeaders& headers = request.headers; @@ -1254,20 +1253,15 @@ util::Optional websocket::make_http_response(const HTTPRequest& re return do_make_http_response(request, sec_websocket_protocol, ec); } -const std::error_category& websocket::websocket_close_status_category() noexcept -{ - static const CloseStatusErrorCategory category = {}; - return category; -} - -std::error_code websocket::make_error_code(ErrorCodes::Error error) noexcept +const std::error_category& websocket::error_category() noexcept { - return std::error_code{error, realm::sync::websocket::websocket_close_status_category()}; + return g_error_category; } -const std::error_category& websocket::error_category() noexcept +const std::error_category& websocket::websocket_close_status_category() noexcept { - return g_error_category; + static const CloseStatusErrorCategory category = {}; + return category; } std::error_code websocket::make_error_code(Error error_code) noexcept diff --git a/src/realm/sync/network/websocket.hpp b/src/realm/sync/network/websocket.hpp index ae7835d8667..1079166cd70 100644 --- a/src/realm/sync/network/websocket.hpp +++ b/src/realm/sync/network/websocket.hpp @@ -18,7 +18,7 @@ class Config { virtual ~Config() {} /// The Socket uses the caller supplied logger for logging. - virtual const std::shared_ptr& websocket_get_logger() noexcept = 0; + virtual util::Logger& websocket_get_logger() noexcept = 0; /// The Socket needs random numbers to satisfy the Websocket protocol. /// The caller must supply a random number generator. @@ -168,10 +168,6 @@ class Socket { /// initiate_server_handshake(). void stop() noexcept; - /// Specifies an alternate status code for the handshake response to simulate - /// failures returned from the server. - void force_handshake_response_for_testing(int status_code, std::string body = ""); - private: class Impl; std::unique_ptr m_impl; @@ -201,7 +197,6 @@ enum class Error { bad_response_200_ok, bad_response_3xx_redirection, bad_response_301_moved_permanently, - bad_response_308_permanent_redirect, bad_response_4xx_client_errors, bad_response_401_unauthorized, bad_response_403_forbidden, @@ -219,8 +214,6 @@ enum class Error { const std::error_category& websocket_close_status_category() noexcept; -std::error_code make_error_code(ErrorCodes::Error error) noexcept; - const std::error_category& error_category() noexcept; std::error_code make_error_code(Error) noexcept; diff --git a/src/realm/sync/noinst/client_history_impl.cpp b/src/realm/sync/noinst/client_history_impl.cpp index 725d5fe76c6..b2b0190a712 100644 --- a/src/realm/sync/noinst/client_history_impl.cpp +++ b/src/realm/sync/noinst/client_history_impl.cpp @@ -229,7 +229,7 @@ util::UniqueFunction ClientReplication::make_wr } void ClientHistory::get_status(version_type& current_client_version, SaltedFileIdent& client_file_ident, - SyncProgress& progress, bool* has_pending_client_reset) const + SyncProgress& progress) const { TransactionRef rt = m_db->start_read(); // Throws version_type current_client_version_2 = rt->get_version(); @@ -262,10 +262,6 @@ void ClientHistory::get_status(version_type& current_client_version, SaltedFileI REALM_ASSERT(current_client_version >= s_initial_version + 0); if (current_client_version == s_initial_version + 0) current_client_version = 0; - - if (has_pending_client_reset) { - *has_pending_client_reset = _impl::client_reset::has_pending_reset(rt).has_value(); - } } @@ -383,18 +379,14 @@ void ClientHistory::find_uploadable_changesets(UploadCursor& upload_progress, ve void ClientHistory::integrate_server_changesets( const SyncProgress& progress, const std::uint_fast64_t* downloadable_bytes, util::Span incoming_changesets, VersionInfo& version_info, DownloadBatchState batch_state, - util::Logger& logger, const TransactionRef& transact, - util::UniqueFunction)> run_in_write_tr, + util::Logger& logger, util::UniqueFunction)> run_in_write_tr, SyncTransactReporter* transact_reporter) { REALM_ASSERT(incoming_changesets.size() != 0); - REALM_ASSERT( - (transact->get_transact_stage() == DB::transact_Writing && batch_state != DownloadBatchState::SteadyState) || - (transact->get_transact_stage() == DB::transact_Reading && batch_state == DownloadBatchState::SteadyState)); std::vector changesets; changesets.resize(incoming_changesets.size()); // Throws - // Parse incoming changesets without holding the write lock unless 'transact' is specified. + // Parse incoming changesets without holding the write lock. try { for (std::size_t i = 0; i < incoming_changesets.size(); ++i) { const RemoteChangeset& changeset = incoming_changesets[i]; @@ -410,16 +402,12 @@ void ClientHistory::integrate_server_changesets( VersionID new_version{0, 0}; auto num_changesets = incoming_changesets.size(); util::Span changesets_to_integrate(changesets); - const bool allow_lock_release = batch_state == DownloadBatchState::SteadyState; // Ideally, this loop runs only once, but it can run up to `incoming_changesets.size()` times, depending on the - // number of times the sync client yields the write lock to allow the user to commit their changes. - // In each iteration, at least one changeset is transformed and committed. - // In FLX, all changesets are committed at once in the bootstrap phase (i.e, in one iteration). + // number of times the sync client yields the write lock to allow the user to commit their changes. In each + // iteration, at least one changeset is transformed and committed. while (!changesets_to_integrate.empty()) { - if (transact->get_transact_stage() == DB::transact_Reading) { - transact->promote_to_write(); // Throws - } + TransactionRef transact = m_db->start_write(); // Throws VersionID old_version = transact->get_version_of_current_transaction(); version_type local_version = old_version.version; auto sync_file_id = transact->get_sync_file_id(); @@ -430,7 +418,7 @@ void ClientHistory::integrate_server_changesets( std::uint64_t downloaded_bytes_in_transaction = 0; auto changesets_transformed_count = transform_and_apply_server_changesets( - changesets_to_integrate, transact, logger, downloaded_bytes_in_transaction, allow_lock_release); + changesets_to_integrate, transact, logger, downloaded_bytes_in_transaction); // downloaded_bytes always contains the total number of downloaded bytes // from the Realm. downloaded_bytes must be persisted in the Realm, since @@ -479,14 +467,7 @@ void ClientHistory::integrate_server_changesets( // this transaction as we already did it. REALM_ASSERT(!m_applying_server_changeset); m_applying_server_changeset = true; - // Commit and continue to write if in bootstrap phase and there are still changes to integrate. - if (batch_state == DownloadBatchState::MoreToCome || - (batch_state == DownloadBatchState::LastInBatch && !changesets_to_integrate.empty())) { - new_version = transact->commit_and_continue_writing(); // Throws - } - else { - new_version = transact->commit_and_continue_as_read(); // Throws - } + new_version = transact->commit_and_continue_as_read(); // Throws if (transact_reporter) { transact_reporter->report_sync_transact(old_version, new_version); // Throws @@ -496,9 +477,6 @@ void ClientHistory::integrate_server_changesets( } REALM_ASSERT(new_version.version > 0); - REALM_ASSERT( - (batch_state == DownloadBatchState::MoreToCome && transact->get_transact_stage() == DB::transact_Writing) || - (batch_state != DownloadBatchState::MoreToCome && transact->get_transact_stage() == DB::transact_Reading)); version_info.realm_version = new_version.version; version_info.sync_version = {new_version.version, 0}; } @@ -506,7 +484,7 @@ void ClientHistory::integrate_server_changesets( size_t ClientHistory::transform_and_apply_server_changesets(util::Span changesets_to_integrate, TransactionRef transact, util::Logger& logger, - std::uint64_t& downloaded_bytes, bool allow_lock_release) + std::uint64_t& downloaded_bytes) { REALM_ASSERT(transact->get_transact_stage() == DB::transact_Writing); @@ -551,8 +529,7 @@ size_t ClientHistory::transform_and_apply_server_changesets(util::Spanoriginal_changeset_size; - return !(m_db->other_writers_waiting_for_lock() && - transact->get_commit_size() >= commit_byte_size_limit && allow_lock_release); + return !(m_db->other_writers_waiting_for_lock() && transact->get_commit_size() >= commit_byte_size_limit); }; auto changesets_transformed_count = transformer.transform_remote_changesets(*this, sync_file_id, local_version, changesets_to_integrate, @@ -786,7 +763,7 @@ void ClientHistory::add_sync_history_entry(const HistoryEntry& entry) void ClientHistory::update_sync_progress(const SyncProgress& progress, const std::uint_fast64_t* downloadable_bytes, - TransactionRef) + TransactionRef wt) { Array& root = m_arrays->root; @@ -835,7 +812,16 @@ void ClientHistory::update_sync_progress(const SyncProgress& progress, const std root.set(s_progress_upload_server_version_iip, RefOrTagged::make_tagged(progress.upload.last_integrated_server_version)); // Throws } - + if (previous_upload_client_version < progress.upload.client_version) { + // This is part of the client reset cycle detection. + // A client reset operation will write a flag to an internal table indicating that + // the changes there are a result of a successful reset. However, it is not possible to + // know if a recovery has been successful until the changes have been acknowledged by the + // server. The situation we want to avoid is that a recovery itself causes another reset + // which creates a reset cycle. However, at this point, upload progress has been made + // and we can remove the cycle detection flag if there is one. + _impl::client_reset::remove_pending_client_resets(wt); + } if (downloadable_bytes) { root.set(s_progress_downloadable_bytes_iip, RefOrTagged::make_tagged(*downloadable_bytes)); // Throws diff --git a/src/realm/sync/noinst/client_history_impl.hpp b/src/realm/sync/noinst/client_history_impl.hpp index 16cac907fd1..f7bd1ec1109 100644 --- a/src/realm/sync/noinst/client_history_impl.hpp +++ b/src/realm/sync/noinst/client_history_impl.hpp @@ -140,8 +140,8 @@ class ClientHistory final : public _impl::History, public TransformHistory { /// The returned SyncProgress is the one that was last stored by /// set_sync_progress(), or `SyncProgress{}` if set_sync_progress() has /// never been called. - void get_status(version_type& current_client_version, SaltedFileIdent& client_file_ident, SyncProgress& progress, - bool* has_pending_client_reset = nullptr) const; + void get_status(version_type& current_client_version, SaltedFileIdent& client_file_ident, + SyncProgress& progress) const; /// Stores the server assigned client file identifier in the associated /// Realm file, such that it is available via get_status() during future @@ -246,18 +246,13 @@ class ClientHistory final : public _impl::History, public TransformHistory { /// about byte-level progress, this function updates the persistent record /// of the estimate of the number of remaining bytes to be downloaded. /// - /// \param transact If specified, it is a transaction to be used to commit - /// the server changesets after they were transformed. - /// Note: In FLX, the transaction is left in reading state when bootstrap ends. - /// In all other cases, the transaction is left in reading state when the function returns. - /// /// \param transact_reporter An optional callback which will be called with the /// version immediately processing the sync transaction and that of the sync /// transaction. void integrate_server_changesets( const SyncProgress& progress, const std::uint_fast64_t* downloadable_bytes, util::Span changesets, VersionInfo& new_version, DownloadBatchState download_type, - util::Logger&, const TransactionRef& transact, + util::Logger&, util::UniqueFunction)> run_in_write_tr = nullptr, SyncTransactReporter* transact_reporter = nullptr); @@ -408,6 +403,7 @@ class ClientHistory final : public _impl::History, public TransformHistory { void initialize(DB& db) noexcept { + REALM_ASSERT(!m_db); m_db = &db; } @@ -421,8 +417,7 @@ class ClientHistory final : public _impl::History, public TransformHistory { version_type end_version) const noexcept; size_t transform_and_apply_server_changesets(util::Span changesets_to_integrate, TransactionRef, - util::Logger&, std::uint64_t& downloaded_bytes, - bool allow_lock_release); + util::Logger&, std::uint64_t& downloaded_bytes); void prepare_for_write(); Replication::version_type add_changeset(BinaryData changeset, BinaryData sync_changeset); diff --git a/src/realm/sync/noinst/client_impl_base.cpp b/src/realm/sync/noinst/client_impl_base.cpp index 29e7805e3c9..6f27348a34d 100644 --- a/src/realm/sync/noinst/client_impl_base.cpp +++ b/src/realm/sync/noinst/client_impl_base.cpp @@ -43,37 +43,6 @@ using OutputBuffer = ClientImpl::OutputBuffer; using ReceivedChangesets = ClientProtocol::ReceivedChangesets; // clang-format on -void ErrorTryAgainBackoffInfo::update(const ProtocolErrorInfo& info) -{ - if (triggering_error && static_cast(info.raw_error_code) == *triggering_error) { - return; - } - - delay_info = info.resumption_delay_interval.value_or(ResumptionDelayInfo{}); - cur_delay_interval = util::none; - triggering_error = static_cast(info.raw_error_code); -} - -void ErrorTryAgainBackoffInfo::reset() -{ - triggering_error = util::none; - cur_delay_interval = util::none; - delay_info = ResumptionDelayInfo{}; -} - -std::chrono::milliseconds ErrorTryAgainBackoffInfo::delay_interval() -{ - if (!cur_delay_interval) { - cur_delay_interval = delay_info.resumption_delay_interval; - return *cur_delay_interval; - } - if (*cur_delay_interval >= delay_info.max_resumption_delay_interval) { - return delay_info.max_resumption_delay_interval; - } - *cur_delay_interval *= delay_info.resumption_delay_backoff_multiplier; - return *cur_delay_interval; -} - bool ClientImpl::decompose_server_url(const std::string& url, ProtocolEnvelope& protocol, std::string& address, port_type& port, std::string& path) const { @@ -141,7 +110,13 @@ ClientImpl::ClientImpl(ClientConfig config) , m_disable_upload_compaction{config.disable_upload_compaction} , m_fix_up_object_ids{config.fix_up_object_ids} , m_roundtrip_time_handler{std::move(config.roundtrip_time_handler)} - , m_socket_provider{std::move(config.socket_provider)} + , m_user_agent_string{make_user_agent_string(config)} // Throws + , m_socket_provider{[&]() -> std::shared_ptr { + if (config.socket_provider) + return config.socket_provider; + + return std::make_shared(logger_ptr, get_user_agent_string()); + }()} , m_client_protocol{} // Throws , m_one_connection_per_session{config.one_connection_per_session} , m_random{} @@ -176,6 +151,7 @@ ClientImpl::ClientImpl(ClientConfig config) config.disable_upload_compaction); // Throws logger.debug("Config param: disable_sync_to_disk = %1", config.disable_sync_to_disk); // Throws + logger.debug("User agent string: '%1'", get_user_agent_string()); if (config.reconnect_mode != ReconnectMode::normal) { logger.warn("Testing/debugging feature 'nonnormal reconnect mode' enabled - " @@ -187,8 +163,6 @@ ClientImpl::ClientImpl(ClientConfig config) "never do this in production!"); } - REALM_ASSERT_EX(m_socket_provider, "Must provide socket provider in sync Client config"); - if (m_one_connection_per_session) { // FIXME: Re-enable this warning when the load balancer is able to handle // multiplexing. @@ -211,45 +185,29 @@ ClientImpl::ClientImpl(ClientConfig config) return; else if (!status.is_ok()) throw ExceptionForStatus(status); + actualize_and_finalize_session_wrappers(); // Throws }); } -void ClientImpl::post(SyncSocketProvider::FunctionHandler&& handler) +std::string ClientImpl::make_user_agent_string(ClientConfig& config) { - REALM_ASSERT(m_socket_provider); - { - std::lock_guard lock(m_drain_mutex); - ++m_outstanding_posts; - m_drained = false; - } - m_socket_provider->post([handler = std::move(handler), this](Status status) { - handler(status); - - std::lock_guard lock(m_drain_mutex); - --m_outstanding_posts; - m_drain_cv.notify_all(); - }); + std::string platform_info = std::move(config.user_agent_platform_info); + if (platform_info.empty()) + platform_info = util::get_platform_info(); // Throws + std::ostringstream out; + out << "RealmSync/" REALM_VERSION_STRING " (" << platform_info << ")"; // Throws + if (!config.user_agent_application_info.empty()) + out << " " << config.user_agent_application_info; // Throws + return out.str(); // Throws } -void ClientImpl::drain_connections() +void ClientImpl::post(SyncSocketProvider::FunctionHandler&& handler) { - logger.debug("Draining connections during sync client shutdown"); - for (auto& server_slot_pair : m_server_slots) { - auto& server_slot = server_slot_pair.second; - - if (server_slot.connection) { - auto& conn = server_slot.connection; - conn->force_close(); - } - else { - for (auto& conn_pair : server_slot.alt_connections) { - conn_pair.second->force_close(); - } - } - } + REALM_ASSERT(m_socket_provider); + m_socket_provider->post(std::move(handler)); } @@ -257,34 +215,15 @@ SyncSocketProvider::SyncTimer ClientImpl::create_timer(std::chrono::milliseconds SyncSocketProvider::FunctionHandler&& handler) { REALM_ASSERT(m_socket_provider); - { - std::lock_guard lock(m_drain_mutex); - ++m_outstanding_posts; - m_drained = false; - } - return m_socket_provider->create_timer(delay, [handler = std::move(handler), this](Status status) { - handler(status); - - std::lock_guard lock(m_drain_mutex); - --m_outstanding_posts; - m_drain_cv.notify_all(); - }); + return m_socket_provider->create_timer(delay, std::move(handler)); } - ClientImpl::SyncTrigger ClientImpl::create_trigger(SyncSocketProvider::FunctionHandler&& handler) { REALM_ASSERT(m_socket_provider); - return std::make_unique>(this, std::move(handler)); + return std::make_unique>(m_socket_provider.get(), std::move(handler)); } -Connection::~Connection() -{ - if (m_websocket_sentinel) { - m_websocket_sentinel->destroyed = true; - m_websocket_sentinel.reset(); - } -} void Connection::activate() { @@ -317,6 +256,7 @@ void Connection::activate_session(std::unique_ptr sess) void Connection::initiate_session_deactivation(Session* sess) { + REALM_ASSERT(m_on_idle); REALM_ASSERT(&sess->m_conn == this); if (REALM_UNLIKELY(--m_num_active_sessions == 0)) { if (m_activated && m_state == ConnectionState::disconnected) @@ -377,25 +317,6 @@ void Connection::cancel_reconnect_delay() } -void Connection::force_close() -{ - if (m_disconnect_delay_in_progress || m_reconnect_delay_in_progress) { - m_reconnect_disconnect_timer.reset(); - m_disconnect_delay_in_progress = false; - m_reconnect_delay_in_progress = false; - } - - REALM_ASSERT(m_num_active_unsuspended_sessions == 0); - REALM_ASSERT(m_num_active_sessions == 0); - if (m_state == ConnectionState::disconnected) { - return; - } - - voluntary_disconnect(); - logger.info("Force disconnected"); -} - - void Connection::websocket_connected_handler(const std::string& protocol) { if (!protocol.empty()) { @@ -434,104 +355,104 @@ void Connection::websocket_connected_handler(const std::string& protocol) } -bool Connection::websocket_binary_message_received(util::Span data) +void Connection::websocket_read_or_write_error_handler(std::error_code ec) { - std::error_code ec; - using sf = SimulatedFailure; - if (sf::trigger(sf::sync_client__read_head, ec)) { - read_or_write_error(ec, "simulated read error"); - return bool(m_websocket); + read_or_write_error(ec); // Throws +} + + +void Connection::websocket_handshake_error_handler(std::error_code ec, const std::string_view* body) +{ + bool is_fatal; + if (ec == websocket::Error::bad_response_3xx_redirection || + ec == websocket::Error::bad_response_301_moved_permanently || + ec == websocket::Error::bad_response_401_unauthorized || + ec == websocket::Error::bad_response_5xx_server_error || + ec == websocket::Error::bad_response_500_internal_server_error || + ec == websocket::Error::bad_response_502_bad_gateway || + ec == websocket::Error::bad_response_503_service_unavailable || + ec == websocket::Error::bad_response_504_gateway_timeout) { + is_fatal = false; + m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_nonfatal_error; + } + else { + is_fatal = true; + m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_fatal_error; + if (body) { + std::string_view identifier = "REALM_SYNC_PROTOCOL_MISMATCH"; + auto i = body->find(identifier); + if (i != std::string_view::npos) { + std::string_view rest = body->substr(i + identifier.size()); + // FIXME: Use std::string_view::begins_with() in C++20. + auto begins_with = [](std::string_view string, std::string_view prefix) { + return (string.size() >= prefix.size() && + std::equal(string.data(), string.data() + prefix.size(), prefix.data())); + }; + if (begins_with(rest, ":CLIENT_TOO_OLD")) { + ec = ClientError::client_too_old_for_server; + } + else if (begins_with(rest, ":CLIENT_TOO_NEW")) { + ec = ClientError::client_too_new_for_server; + } + else { + // Other more complicated forms of mismatch + ec = ClientError::protocol_mismatch; + } + } + } } - handle_message_received(data); - return bool(m_websocket); + close_due_to_client_side_error(ec, std::nullopt, is_fatal); // Throws } -void Connection::websocket_error_handler() +void Connection::websocket_protocol_error_handler(std::error_code ec) { - m_websocket_error_received = true; + m_reconnect_info.m_reason = ConnectionTerminationReason::websocket_protocol_violation; + bool is_fatal = true; // A WebSocket protocol violation is a fatal error + close_due_to_client_side_error(ec, std::nullopt, is_fatal); // Throws } -bool Connection::websocket_closed_handler(bool was_clean, Status status) +bool Connection::websocket_binary_message_received(util::Span data) { - logger.info("Closing the websocket with status='%1', was_clean='%2'", status, was_clean); - // Return early. - if (status.is_ok()) { + std::error_code ec; + using sf = SimulatedFailure; + if (sf::trigger(sf::sync_client__read_head, ec)) { + read_or_write_error(ec); return bool(m_websocket); } - auto&& status_code = status.code(); - std::error_code error_code{static_cast(status_code), websocket::websocket_close_status_category()}; + handle_message_received(data); + return bool(m_websocket); +} - // TODO: Use a switch statement once websocket errors have their own category in exception unification. - if (status_code == ErrorCodes::ResolveFailed || status_code == ErrorCodes::ConnectionFailed) { - m_reconnect_info.m_reason = ConnectionTerminationReason::connect_operation_failed; - constexpr bool try_again = true; - involuntary_disconnect(SessionErrorInfo{error_code, try_again}); // Throws - } - else if (status_code == ErrorCodes::ReadError || status_code == ErrorCodes::WriteError) { - read_or_write_error(error_code, status.reason()); - } - else if (status_code == ErrorCodes::WebSocket_GoingAway || status_code == ErrorCodes::WebSocket_ProtocolError || - status_code == ErrorCodes::WebSocket_UnsupportedData || status_code == ErrorCodes::WebSocket_Reserved || - status_code == ErrorCodes::WebSocket_InvalidPayloadData || - status_code == ErrorCodes::WebSocket_PolicyViolation || - status_code == ErrorCodes::WebSocket_InavalidExtension) { - m_reconnect_info.m_reason = ConnectionTerminationReason::websocket_protocol_violation; - constexpr bool try_again = true; - SessionErrorInfo error_info{error_code, status.reason(), try_again}; - involuntary_disconnect(std::move(error_info)); - } - else if (status_code == ErrorCodes::WebSocket_MessageTooBig) { + +bool Connection::websocket_close_message_received(std::error_code error_code, StringData message) +{ + if (error_code.category() == websocket::websocket_close_status_category() && error_code.value() != 1005 && + error_code.value() != 1000) { m_reconnect_info.m_reason = ConnectionTerminationReason::websocket_protocol_violation; + constexpr bool try_again = true; - auto ec = make_error_code(ProtocolError::limits_exceeded); - auto message = util::format( - "Sync websocket closed because the server received a message that was too large: %1", status.reason()); - SessionErrorInfo error_info(ec, message, try_again); - error_info.server_requests_action = ProtocolErrorInfo::Action::ClientReset; + SessionErrorInfo error_info{error_code, message, try_again}; + + // If the server sends a websocket close message with code 1009, then it's because we've sent an + // UPLOAD message that is too large for the server to process. Simply disconnecting/reconnecting will not + // be sufficient because when we re-connect we'll just try to send the same bad upload message. + // + // Since the handling of this error happens at a layer below the standard `ERROR` message handling + // we need to synthesize an `ERROR` message-like error info here to client reset when this error + // is received. + if (error_code.value() == 1009) { + error_info.error_code = make_error_code(ProtocolError::limits_exceeded); + error_info.server_requests_action = ProtocolErrorInfo::Action::ClientReset; + error_info.message = util::format( + "Sync websocket closed because the server received a message that was too large: %1", message); + } + involuntary_disconnect(std::move(error_info)); } - else if (status_code == ErrorCodes::WebSocket_TLSHandshakeFailed) { - error_code = ClientError::ssl_server_cert_rejected; - constexpr bool is_fatal = true; - m_reconnect_info.m_reason = ConnectionTerminationReason::ssl_certificate_rejected; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } - else if (status_code == ErrorCodes::WebSocket_Client_Too_Old) { - error_code = ClientError::client_too_old_for_server; - constexpr bool is_fatal = true; - m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_fatal_error; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } - else if (status_code == ErrorCodes::WebSocket_Client_Too_New) { - error_code = ClientError::client_too_new_for_server; - constexpr bool is_fatal = true; - m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_fatal_error; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } - else if (status_code == ErrorCodes::WebSocket_Protocol_Mismatch) { - error_code = ClientError::protocol_mismatch; - constexpr bool is_fatal = true; - m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_fatal_error; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } - else if (status_code == ErrorCodes::WebSocket_Fatal_Error || status_code == ErrorCodes::WebSocket_Forbidden) { - constexpr bool is_fatal = true; - m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_fatal_error; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } - else if (status_code == ErrorCodes::WebSocket_Unauthorized || - status_code == ErrorCodes::WebSocket_MovedPermanently || - status_code == ErrorCodes::WebSocket_InternalServerError || - status_code == ErrorCodes::WebSocket_AbnormalClosure || - status_code == ErrorCodes::WebSocket_Retry_Error) { - constexpr bool is_fatal = false; - m_reconnect_info.m_reason = ConnectionTerminationReason::http_response_says_nonfatal_error; - close_due_to_client_side_error(error_code, status.reason(), is_fatal); // Throws - } return bool(m_websocket); } @@ -571,6 +492,7 @@ void Connection::initiate_reconnect_wait() } else { // Compute a new reconnect delay + bool zero_delay = false; switch (m_client.get_reconnect_mode()) { case ReconnectMode::normal: @@ -614,8 +536,8 @@ void Connection::initiate_reconnect_wait() delay = max_delay; break; case ConnectionTerminationReason::server_said_try_again_later: + delay = max_delay; record_delay_as_zero = true; - delay = m_reconnect_info.m_try_again_delay_info.delay_interval().count(); break; case ConnectionTerminationReason::ssl_certificate_rejected: case ConnectionTerminationReason::ssl_protocol_violation: @@ -709,52 +631,6 @@ void Connection::handle_reconnect_wait(Status status) initiate_reconnect(); // Throws } -struct Connection::WebSocketObserverShim : public sync::WebSocketObserver { - explicit WebSocketObserverShim(Connection* conn) - : conn(conn) - , sentinel(conn->m_websocket_sentinel) - { - } - - Connection* conn; - util::bind_ptr sentinel; - - void websocket_connected_handler(const std::string& protocol) override - { - if (sentinel->destroyed) { - return; - } - - return conn->websocket_connected_handler(protocol); - } - - void websocket_error_handler() override - { - if (sentinel->destroyed) { - return; - } - - conn->websocket_error_handler(); - } - - bool websocket_binary_message_received(util::Span data) override - { - if (sentinel->destroyed) { - return false; - } - - return conn->websocket_binary_message_received(data); - } - - bool websocket_closed_handler(bool was_clean, Status status) override - { - if (sentinel->destroyed) { - return true; - } - - return conn->websocket_closed_handler(was_clean, std::move(status)); - } -}; void Connection::initiate_reconnect() { @@ -762,10 +638,6 @@ void Connection::initiate_reconnect() m_state = ConnectionState::connecting; report_connection_state_change(ConnectionState::connecting); // Throws - if (m_websocket_sentinel) { - m_websocket_sentinel->destroyed = true; - } - m_websocket_sentinel = util::make_bind(); m_websocket.reset(); // In most cases, the reconnect delay will be counting from the point in @@ -798,22 +670,20 @@ void Connection::initiate_reconnect() } } - m_websocket_error_received = false; - m_websocket = - m_client.m_socket_provider->connect(std::make_unique(this), - WebSocketEndpoint{ - m_address, - m_port, - get_http_request_path(), - std::move(sec_websocket_protocol), - is_ssl(m_protocol_envelope), - /// DEPRECATED - The following will be removed in a future release - {m_custom_http_headers.begin(), m_custom_http_headers.end()}, - m_verify_servers_ssl_certificate, - m_ssl_trust_certificate_path, - m_ssl_verify_callback, - m_proxy_config, - }); + m_websocket = m_client.m_socket_provider->connect( + this, WebSocketEndpoint{ + m_address, + m_port, + get_http_request_path(), + std::move(sec_websocket_protocol), + is_ssl(m_protocol_envelope), + /// DEPRECATED - The following will be removed in a future release + {m_custom_http_headers.begin(), m_custom_http_headers.end()}, + m_verify_servers_ssl_certificate, + m_ssl_trust_certificate_path, + m_ssl_verify_callback, + m_proxy_config, + }); } @@ -939,9 +809,9 @@ void Connection::initiate_ping_delay(milliseconds_type now) else if (!status.is_ok()) throw ExceptionForStatus(status); - handle_ping_delay(); // Throws - }); // Throws - logger.debug("Will emit a ping in %1 milliseconds", delay); // Throws + handle_ping_delay(); // Throws + }); // Throws + logger.debug("Will emit a ping in %1 milliseconds", delay); // Throws } @@ -990,14 +860,7 @@ void Connection::handle_pong_timeout() void Connection::initiate_write_message(const OutputBuffer& out, Session* sess) { - // Stop sending messages if an websocket error was received. - if (m_websocket_error_received) - return; - - m_websocket->async_write_binary(out.as_span(), [this, sentinel = m_websocket_sentinel](Status status) { - if (sentinel->destroyed) { - return; - } + m_websocket->async_write_binary(util::Span{out.data(), out.size()}, [this](Status status) { if (status == ErrorCodes::OperationAborted) return; else if (!status.is_ok()) @@ -1081,10 +944,7 @@ void Connection::send_ping() void Connection::initiate_write_ping(const OutputBuffer& out) { - m_websocket->async_write_binary(out.as_span(), [this, sentinel = m_websocket_sentinel](Status status) { - if (sentinel->destroyed) { - return; - } + m_websocket->async_write_binary(util::Span{out.data(), out.size()}, [this](Status status) { if (status == ErrorCodes::OperationAborted) return; else if (!status.is_ok()) @@ -1153,11 +1013,41 @@ void Connection::handle_disconnect_wait(Status status) } -void Connection::read_or_write_error(std::error_code ec, std::string_view msg) +void Connection::websocket_connect_error_handler(std::error_code ec) +{ + m_reconnect_info.m_reason = ConnectionTerminationReason::connect_operation_failed; + constexpr bool try_again = true; + involuntary_disconnect(SessionErrorInfo{ec, try_again}); // Throws +} + +void Connection::websocket_ssl_handshake_error_handler(std::error_code ec) +{ + logger.error("SSL handshake failed: %1", ec.message()); // Throws + // FIXME: Some error codes (those from OpenSSL) most likely indicate a + // fatal error (SSL protocol violation), but other errors codes + // (read/write error from underlying socket) most likely indicate a + // nonfatal error. + bool is_fatal = false; + std::error_code ec2; + if (ec == network::ssl::Errors::certificate_rejected) { + m_reconnect_info.m_reason = ConnectionTerminationReason::ssl_certificate_rejected; + ec2 = ClientError::ssl_server_cert_rejected; + is_fatal = true; + } + else { + m_reconnect_info.m_reason = ConnectionTerminationReason::read_or_write_error; + ec2 = ec; + is_fatal = false; + } + close_due_to_client_side_error(ec2, std::nullopt, is_fatal); // Throws +} + + +void Connection::read_or_write_error(std::error_code ec) { m_reconnect_info.m_reason = ConnectionTerminationReason::read_or_write_error; bool is_fatal = false; - close_due_to_client_side_error(ec, msg, is_fatal); // Throws + close_due_to_client_side_error(ec, std::nullopt, is_fatal); // Throws } @@ -1169,6 +1059,15 @@ void Connection::close_due_to_protocol_error(std::error_code ec, std::optional msg, bool is_fatal) @@ -1195,8 +1094,6 @@ void Connection::close_due_to_server_side_error(ProtocolError error_code, const m_reconnect_info.m_reason = ConnectionTerminationReason::server_said_do_not_reconnect; } - m_reconnect_info.m_try_again_delay_info.update(info); - // When the server asks us to reconnect later, it is important to make the // reconnect delay start at the time of the reception of the ERROR message, // rather than at the initiation of the connection, as is usually the @@ -1250,8 +1147,6 @@ void Connection::disconnect(const SessionErrorInfo& info) m_heartbeat_timer.reset(); m_previous_ping_rtt = 0; - m_websocket_sentinel->destroyed = true; - m_websocket_sentinel.reset(); m_websocket.reset(); m_input_body_buffer.reset(); m_sending_session = nullptr; @@ -1578,9 +1473,8 @@ void Session::integrate_changesets(ClientReplication& repl, const SyncProgress& } std::vector pending_compensating_write_errors; - auto transact = get_db()->start_read(); history.integrate_server_changesets( - progress, &downloadable_bytes, received_changesets, version_info, download_batch_state, logger, transact, + progress, &downloadable_bytes, received_changesets, version_info, download_batch_state, logger, [&](const TransactionRef&, util::Span changesets) { gather_pending_compensating_writes(changesets, &pending_compensating_write_errors); }, @@ -1677,7 +1571,6 @@ void Session::activate() logger.debug("Activating"); // Throws - bool has_pending_client_reset = false; if (REALM_LIKELY(!get_client().is_dry_run())) { // The reason we need a mutable reference from get_client_reset_config() is because we // don't want the session to keep a strong reference to the client_reset_config->fresh_copy @@ -1704,9 +1597,8 @@ void Session::activate() } if (!m_client_reset_operation) { - const ClientReplication& repl = access_realm(); // Throws - repl.get_history().get_status(m_last_version_available, m_client_file_ident, m_progress, - &has_pending_client_reset); // Throws + const ClientReplication& repl = access_realm(); // Throws + repl.get_history().get_status(m_last_version_available, m_client_file_ident, m_progress); // Throws } } logger.debug("client_file_ident = %1, client_file_ident_salt = %2", m_client_file_ident.ident, @@ -1736,10 +1628,6 @@ void Session::activate() on_suspended(SessionErrorInfo{error.code(), false}); m_conn.one_less_active_unsuspended_session(); // Throws } - - if (has_pending_client_reset) { - handle_pending_client_reset_acknowledgement(); - } } @@ -2272,9 +2160,7 @@ std::error_code Session::receive_ident_message(SaltedFileIdent client_file_ident logger.debug("Client reset is completed, path=%1", get_realm_path()); // Throws SaltedFileIdent client_file_ident; - bool has_pending_client_reset = false; - repl.get_history().get_status(m_last_version_available, client_file_ident, m_progress, - &has_pending_client_reset); // Throws + repl.get_history().get_status(m_last_version_available, client_file_ident, m_progress); // Throws REALM_ASSERT_EX(m_client_file_ident.ident == client_file_ident.ident, m_client_file_ident.ident, client_file_ident.ident); REALM_ASSERT_EX(m_client_file_ident.salt == client_file_ident.salt, m_client_file_ident.salt, @@ -2295,10 +2181,6 @@ std::error_code Session::receive_ident_message(SaltedFileIdent client_file_ident REALM_ASSERT_EX(m_last_version_selected_for_upload == 0, m_last_version_selected_for_upload); get_transact_reporter()->report_sync_transact(client_reset_old_version, client_reset_new_version); - - if (has_pending_client_reset) { - handle_pending_client_reset_acknowledgement(); - } return true; }; // if a client reset happens, it will take care of setting the file ident @@ -2592,32 +2474,41 @@ std::error_code Session::receive_test_command_response(request_ident_type ident, void Session::begin_resumption_delay(const ProtocolErrorInfo& error_info) { REALM_ASSERT(!m_try_again_activation_timer); - - m_try_again_delay_info.update(error_info); - auto try_again_interval = m_try_again_delay_info.delay_interval(); - if (ProtocolError(error_info.raw_error_code) == ProtocolError::session_closed) { + if (error_info.resumption_delay_interval) { + m_try_again_delay_info = *error_info.resumption_delay_interval; + } + if (!m_current_try_again_delay_interval || + (m_try_again_error_code && *m_try_again_error_code != ProtocolError(error_info.raw_error_code))) { + m_current_try_again_delay_interval = m_try_again_delay_info.resumption_delay_interval; + } + else if (ProtocolError(error_info.raw_error_code) == ProtocolError::session_closed) { // FIXME With compensating writes the server sends this error after completing a bootstrap. Doing the normal // backoff behavior would result in waiting up to 5 minutes in between each query change which is // not acceptable latency. So for this error code alone, we hard-code a 1 second retry interval. - try_again_interval = std::chrono::milliseconds{1000}; - } - logger.debug("Will attempt to resume session after %1 milliseconds", try_again_interval.count()); - m_try_again_activation_timer = get_client().create_timer(try_again_interval, [this](Status status) { - if (status == ErrorCodes::OperationAborted) - return; - else if (!status.is_ok()) - throw ExceptionForStatus(status); - - m_try_again_activation_timer.reset(); - cancel_resumption_delay(); - }); + m_current_try_again_delay_interval = std::chrono::milliseconds{1000}; + } + m_try_again_error_code = ProtocolError(error_info.raw_error_code); + logger.debug("Will attempt to resume session after %1 milliseconds", m_current_try_again_delay_interval->count()); + m_try_again_activation_timer = + get_client().create_timer(*m_current_try_again_delay_interval, [this](Status status) { + if (status == ErrorCodes::OperationAborted) + return; + else if (!status.is_ok()) + throw ExceptionForStatus(status); + + m_try_again_activation_timer.reset(); + if (m_current_try_again_delay_interval < m_try_again_delay_info.max_resumption_delay_interval) { + *m_current_try_again_delay_interval *= m_try_again_delay_info.resumption_delay_backoff_multiplier; + } + cancel_resumption_delay(); + }); } void Session::clear_resumption_delay_state() { if (m_try_again_activation_timer) { logger.debug("Clearing resumption delay state after successful download"); - m_try_again_delay_info.reset(); + m_current_try_again_delay_interval = util::none; } } diff --git a/src/realm/sync/noinst/client_impl_base.hpp b/src/realm/sync/noinst/client_impl_base.hpp index ba709a6e6ac..f17123ce28d 100644 --- a/src/realm/sync/noinst/client_impl_base.hpp +++ b/src/realm/sync/noinst/client_impl_base.hpp @@ -57,16 +57,6 @@ class SessionWrapperStack { SessionWrapper* m_back = nullptr; }; -struct ErrorTryAgainBackoffInfo { - void update(const ProtocolErrorInfo& info); - void reset(); - std::chrono::milliseconds delay_interval(); - - ResumptionDelayInfo delay_info; - util::Optional cur_delay_interval; - util::Optional triggering_error; -}; - class ClientImpl { public: enum class ConnectionTerminationReason; @@ -117,8 +107,6 @@ class ClientImpl { // true. See receive_pong(). bool m_scheduled_reset = false; - ErrorTryAgainBackoffInfo m_try_again_delay_info; - friend class Connection; }; @@ -136,10 +124,11 @@ class ClientImpl { static constexpr int get_oldest_supported_protocol_version() noexcept; - /// This calls stop() on the socket provider respectively. + // @{ + /// These call stop() and run() on the socket provider respectively. void stop() noexcept; - - void drain(); + void run(); + // @} const std::string& get_user_agent_string() const noexcept; ReconnectMode get_reconnect_mode() const noexcept; @@ -150,7 +139,7 @@ class ClientImpl { void post(SyncSocketProvider::FunctionHandler&& handler); SyncSocketProvider::SyncTimer create_timer(std::chrono::milliseconds delay, SyncSocketProvider::FunctionHandler&& handler); - using SyncTrigger = std::unique_ptr>; + using SyncTrigger = std::unique_ptr>; SyncTrigger create_trigger(SyncSocketProvider::FunctionHandler&& handler); std::mt19937_64& get_random() noexcept; @@ -178,6 +167,7 @@ class ClientImpl { const bool m_fix_up_object_ids; const std::function m_roundtrip_time_handler; const std::string m_user_agent_string; + // This will be updated to the SyncSocketProvider interface once the integration is complete std::shared_ptr m_socket_provider; ClientProtocol m_client_protocol; session_ident_type m_prev_session_ident = 0; @@ -212,18 +202,14 @@ class ClientImpl { // Must be accessed only by event loop thread connection_ident_type m_prev_connection_ident = 0; - std::mutex m_drain_mutex; - std::condition_variable m_drain_cv; - bool m_drained = false; - uint64_t m_outstanding_posts = 0; - uint64_t m_num_connections = 0; - - std::mutex m_mutex; + util::Mutex m_mutex; bool m_stopped = false; // Protected by `m_mutex` bool m_sessions_terminated = false; // Protected by `m_mutex` bool m_actualize_and_finalize_needed = false; // Protected by `m_mutex` + std::atomic m_running{false}; // Debugging facility + // The set of session wrappers that are not yet wrapping a session object, // and are not yet abandoned (still referenced by the application). // @@ -238,7 +224,7 @@ class ClientImpl { SessionWrapperStack m_abandoned_session_wrappers; // Protected by `m_mutex`. - std::condition_variable m_wait_or_client_stopped_cond; + util::CondVar m_wait_or_client_stopped_cond; void register_unactualized_session_wrapper(SessionWrapper*, ServerEndpoint); void register_abandoned_session_wrapper(util::bind_ptr) noexcept; @@ -276,8 +262,7 @@ class ClientImpl { // Destroys the specified connection. void remove_connection(ClientImpl::Connection&) noexcept; - void drain_connections(); - void drain_connections_on_loop(); + static std::string make_user_agent_string(ClientConfig&); session_ident_type get_next_session_ident() noexcept; @@ -325,7 +310,7 @@ enum class ClientImpl::ConnectionTerminationReason { /// occur on behalf of the event loop thread of the associated client object. // TODO: The parent will be updated to WebSocketObserver once the WebSocket integration is complete -class ClientImpl::Connection { +class ClientImpl::Connection final : public WebSocketObserver { public: using connection_ident_type = std::int_fast64_t; using SSLVerifyCallback = SyncConfig::SSLVerifyCallback; @@ -390,8 +375,6 @@ class ClientImpl::Connection { /// activated. void cancel_reconnect_delay(); - void force_close(); - /// Returns zero until the HTTP response is received. After that point in /// time, it returns the negotiated protocol version, which is based on the /// contents of the `Sec-WebSocket-Protocol` header in the HTTP @@ -401,10 +384,23 @@ class ClientImpl::Connection { int get_negotiated_protocol_version() noexcept; // Methods from WebSocketObserver interface for websockets from the Socket Provider - void websocket_connected_handler(const std::string& protocol); - bool websocket_binary_message_received(util::Span data); - void websocket_error_handler(); - bool websocket_closed_handler(bool, Status); + void websocket_connected_handler(const std::string& protocol) override; + bool websocket_binary_message_received(util::Span data) override; + // Will be implemented when the functions below are removed + void websocket_error_handler() override {} + bool websocket_closed_handler(bool, Status) override + { + return false; + } + + /// DEPRECATED - Will be removed in a future release + // Methods from WebsocketObserver that will be going away soon + void websocket_connect_error_handler(std::error_code) override; + void websocket_ssl_handshake_error_handler(std::error_code) override; + void websocket_read_or_write_error_handler(std::error_code) override; + void websocket_handshake_error_handler(std::error_code, const std::string_view*) override; + void websocket_protocol_error_handler(std::error_code) override; + bool websocket_close_message_received(std::error_code error_code, StringData message) override; connection_ident_type get_ident() const noexcept; const ServerEndpoint& get_server_endpoint() const noexcept; @@ -424,11 +420,6 @@ class ClientImpl::Connection { ~Connection(); private: - struct LifecycleSentinel : public util::AtomicRefCountBase { - bool destroyed = false; - }; - struct WebSocketObserverShim; - using ReceivedChangesets = ClientProtocol::ReceivedChangesets; template @@ -477,8 +468,9 @@ class ClientImpl::Connection { void handle_message_received(util::Span data); void initiate_disconnect_wait(); void handle_disconnect_wait(Status status); - void read_or_write_error(std::error_code ec, std::string_view msg); + void read_or_write_error(std::error_code); void close_due_to_protocol_error(std::error_code, std::optional msg = std::nullopt); + void close_due_to_missing_protocol_feature(); void close_due_to_client_side_error(std::error_code, std::optional msg, bool is_fatal); void close_due_to_server_side_error(ProtocolError, const ProtocolErrorInfo& info); void voluntary_disconnect(); @@ -515,7 +507,6 @@ class ClientImpl::Connection { friend class Session; ClientImpl& m_client; - util::bind_ptr m_websocket_sentinel; std::unique_ptr m_websocket; const ProtocolEnvelope m_protocol_envelope; const std::string m_address; @@ -567,8 +558,6 @@ class ClientImpl::Connection { // At least one PING message was sent since connection was established bool m_ping_sent = false; - bool m_websocket_error_received = false; - // The timer will be constructed on demand, and will only be destroyed when // canceling a reconnect or disconnect delay. // @@ -921,8 +910,6 @@ class ClientImpl::Session { // Processes any pending FLX bootstraps, if one exists. Otherwise this is a noop. void process_pending_flx_bootstrap(); - void handle_pending_client_reset_acknowledgement(); - void gather_pending_compensating_writes(util::Span changesets, std::vector* out); void begin_resumption_delay(const ProtocolErrorInfo& error_info); @@ -943,7 +930,9 @@ class ClientImpl::Session { bool m_suspended = false; SyncSocketProvider::SyncTimer m_try_again_activation_timer; - ErrorTryAgainBackoffInfo m_try_again_delay_info; + ResumptionDelayInfo m_try_again_delay_info; + util::Optional m_try_again_error_code; + util::Optional m_current_try_again_delay_interval; // Set to true when download completion is reached. Set to false after a // slow reconnect, such that the upload process will become suspended until @@ -1163,6 +1152,7 @@ inline bool ClientImpl::is_dry_run() const noexcept return m_dry_run; } + inline std::mt19937_64& ClientImpl::get_random() noexcept { return m_random; @@ -1179,7 +1169,6 @@ inline void ClientImpl::ReconnectInfo::reset() noexcept m_time_point = 0; m_delay = 0; m_scheduled_reset = false; - m_try_again_delay_info.reset(); } inline ClientImpl& ClientImpl::Connection::get_client() noexcept @@ -1212,6 +1201,8 @@ inline int ClientImpl::Connection::get_negotiated_protocol_version() noexcept return m_negotiated_protocol_version; } +inline ClientImpl::Connection::~Connection() {} + template void ClientImpl::Connection::for_each_active_session(H handler) { diff --git a/src/realm/sync/noinst/client_reset.cpp b/src/realm/sync/noinst/client_reset.cpp index 17c1cb7c12b..5e47f3dd375 100644 --- a/src/realm/sync/noinst/client_reset.cpp +++ b/src/realm/sync/noinst/client_reset.cpp @@ -329,30 +329,8 @@ void transfer_group(const Transaction& group_src, Transaction& group_dst, util:: } } - // We must re-create any missing objects that are absent in dst before trying to copy - // their properties because creating them may re-create any dangling links which would - // otherwise cause inconsistencies when re-creating lists of links. - for (auto table_key : group_src.get_table_keys()) { - ConstTableRef table_src = group_src.get_table(table_key); - auto table_name = table_src->get_name(); - if (should_skip_table(group_src, table_key) || table_src->is_embedded()) - continue; - TableRef table_dst = group_dst.get_table(table_name); - auto pk_col = table_src->get_primary_key_column(); - REALM_ASSERT(pk_col); - logger.debug("Creating missing objects for table '%1', number of rows = %2, " - "primary_key_col = %3, primary_key_type = %4", - table_name, table_src->size(), pk_col.get_index().val, pk_col.get_type()); - for (const Obj& src : *table_src) { - bool created = false; - table_dst->create_object_with_primary_key(src.get_primary_key(), &created); - if (created) { - logger.debug(" created %1", src.get_primary_key()); - } - } - } - converters::EmbeddedObjectConverter embedded_tracker; + // Now src and dst have identical schemas and no extraneous objects from dst. // There may be missing object from src and the values of existing objects may // still differ. Diff all the values and create missing objects on the fly. @@ -379,11 +357,11 @@ void transfer_group(const Transaction& group_src, Transaction& group_dst, util:: for (const Obj& src : *table_src) { auto src_pk = src.get_primary_key(); - // create the object - it should have been created above. - auto dst = table_dst->get_object_with_primary_key(src_pk); + bool updated = false; + // get or create the object + auto dst = table_dst->create_object_with_primary_key(src_pk, &updated); REALM_ASSERT(dst); - bool updated = false; converter.copy(src, dst, &updated); if (updated) { logger.debug(" updating %1", src_pk); @@ -477,12 +455,6 @@ void track_reset(TransactionRef wt, ClientResyncMode mode) if (mode == ClientResyncMode::Recover || mode == ClientResyncMode::RecoverOrDiscard) { mode_val = 1; // Recover } - - if (table->size() > 1) { - // this may happen if a future version of this code changes the format and expectations around reset metadata. - throw ClientResetFailed( - util::format("Previous client resets detected (%1) but only one is expected.", table->size())); - } table->create_object_with_primary_key(ObjectId::gen(), {{version_col, metadata_version}, {timestamp_col, Timestamp(std::chrono::system_clock::now())}, @@ -498,28 +470,27 @@ static ClientResyncMode reset_precheck_guard(TransactionRef wt, ClientResyncMode switch (previous_reset->type) { case ClientResyncMode::Manual: REALM_UNREACHABLE(); + break; case ClientResyncMode::DiscardLocal: throw ClientResetFailed(util::format("A previous '%1' mode reset from %2 did not succeed, " "giving up on '%3' mode to prevent a cycle", previous_reset->type, previous_reset->time, mode)); case ClientResyncMode::Recover: - switch (mode) { - case ClientResyncMode::Recover: - throw ClientResetFailed(util::format("A previous '%1' mode reset from %2 did not succeed, " - "giving up on '%3' mode to prevent a cycle", - previous_reset->type, previous_reset->time, mode)); - case ClientResyncMode::RecoverOrDiscard: - mode = ClientResyncMode::DiscardLocal; - logger.info("A previous '%1' mode reset from %2 downgrades this mode ('%3') to DiscardLocal", - previous_reset->type, previous_reset->time, mode); - remove_pending_client_resets(wt); - break; - case ClientResyncMode::DiscardLocal: - remove_pending_client_resets(wt); - // previous mode Recover and this mode is Discard, this is not a cycle yet - break; - case ClientResyncMode::Manual: - REALM_UNREACHABLE(); + if (mode == ClientResyncMode::Recover) { + throw ClientResetFailed(util::format("A previous '%1' mode reset from %2 did not succeed, " + "giving up on '%3' mode to prevent a cycle", + previous_reset->type, previous_reset->time, mode)); + } + else if (mode == ClientResyncMode::RecoverOrDiscard) { + mode = ClientResyncMode::DiscardLocal; + logger.info("A previous '%1' mode reset from %2 downgrades this mode ('%3') to DiscardLocal", + previous_reset->type, previous_reset->time, mode); + } + else if (mode == ClientResyncMode::DiscardLocal) { + // previous mode Recover and this mode is Discard, this is not a cycle yet + } + else { + REALM_UNREACHABLE(); } break; case ClientResyncMode::RecoverOrDiscard: diff --git a/src/realm/sync/noinst/client_reset_operation.cpp b/src/realm/sync/noinst/client_reset_operation.cpp index b7d228bcdff..303dfae379d 100644 --- a/src/realm/sync/noinst/client_reset_operation.cpp +++ b/src/realm/sync/noinst/client_reset_operation.cpp @@ -59,44 +59,42 @@ bool ClientResetOperation::finalize(sync::SaltedFileIdent salted_file_ident, syn // only do the reset if there is data to reset // if there is nothing in this Realm, then there is nothing to reset and // sync should be able to continue as normal - auto latest_version = m_db->get_version_id_of_latest_snapshot(); + bool local_realm_exists = m_db->get_version_of_latest_snapshot() != 0; + if (local_realm_exists) { + REALM_ASSERT_EX(m_db_fresh, m_db->get_path(), m_mode); + m_logger.debug("ClientResetOperation::finalize, realm_path = %1, local_realm_exists = %2, mode = %3", + m_db->get_path(), local_realm_exists, m_mode); - bool local_realm_exists = latest_version.version != 0; - m_logger.debug("ClientResetOperation::finalize, realm_path = %1, local_realm_exists = %2, mode = %3", - m_db->get_path(), local_realm_exists, m_mode); - if (!local_realm_exists) { - return false; - } + client_reset::LocalVersionIDs local_version_ids; + auto always_try_clean_up = util::make_scope_exit([&]() noexcept { + clean_up_state(); + }); - REALM_ASSERT_EX(m_db_fresh, m_db->get_path(), m_mode); + std::string local_path = m_db->get_path(); + if (m_notify_before) { + m_notify_before(local_path); + } - client_reset::LocalVersionIDs local_version_ids; - auto always_try_clean_up = util::make_scope_exit([&]() noexcept { - clean_up_state(); - }); + // If m_notify_after is set, pin the previous state to keep it around. + TransactionRef previous_state; + if (m_notify_after) { + previous_state = m_db->start_frozen(); + } + bool did_recover_out = false; + local_version_ids = client_reset::perform_client_reset_diff( + m_db, m_db_fresh, m_salted_file_ident, m_logger, m_mode, m_recovery_is_allowed, &did_recover_out, + sub_store, std::move(on_flx_version_complete)); // throws - if (m_notify_before) { - m_notify_before(latest_version); - } + if (m_notify_after) { + m_notify_after(local_path, previous_state->get_version_of_current_transaction(), did_recover_out); + } - // If m_notify_after is set, pin the previous state to keep it around. - TransactionRef previous_state; - if (m_notify_after) { - previous_state = m_db->start_frozen(); - } - bool did_recover_out = false; - local_version_ids = client_reset::perform_client_reset_diff( - m_db, m_db_fresh, m_salted_file_ident, m_logger, m_mode, m_recovery_is_allowed, &did_recover_out, sub_store, - std::move(on_flx_version_complete)); // throws + m_client_reset_old_version = local_version_ids.old_version; + m_client_reset_new_version = local_version_ids.new_version; - if (m_notify_after) { - m_notify_after(previous_state->get_version_of_current_transaction(), did_recover_out); + return true; } - - m_client_reset_old_version = local_version_ids.old_version; - m_client_reset_new_version = local_version_ids.new_version; - - return true; + return false; } void ClientResetOperation::clean_up_state() noexcept diff --git a/src/realm/sync/noinst/client_reset_operation.hpp b/src/realm/sync/noinst/client_reset_operation.hpp index 2a54dc01d05..26273f02509 100644 --- a/src/realm/sync/noinst/client_reset_operation.hpp +++ b/src/realm/sync/noinst/client_reset_operation.hpp @@ -34,8 +34,8 @@ namespace realm::_impl { // state Realm download. class ClientResetOperation { public: - using CallbackBeforeType = util::UniqueFunction; - using CallbackAfterType = util::UniqueFunction; + using CallbackBeforeType = util::UniqueFunction; + using CallbackAfterType = util::UniqueFunction; ClientResetOperation(util::Logger& logger, DBRef db, DBRef db_fresh, ClientResyncMode mode, CallbackBeforeType notify_before, CallbackAfterType notify_after, bool recovery_is_allowed); diff --git a/src/realm/sync/noinst/server/server.cpp b/src/realm/sync/noinst/server/server.cpp index e8512dfe46c..2d1c3ee179f 100644 --- a/src/realm/sync/noinst/server/server.cpp +++ b/src/realm/sync/noinst/server/server.cpp @@ -1070,15 +1070,13 @@ class ServerImpl : public ServerImplBase, public ServerHistory::Context { class SyncConnection : public websocket::Config { public: - const std::shared_ptr logger_ptr; - util::Logger& logger; + util::PrefixLogger logger; SyncConnection(ServerImpl& serv, std::int_fast64_t id, std::unique_ptr&& socket, std::unique_ptr&& ssl_stream, std::unique_ptr&& read_ahead_buffer, int client_protocol_version, std::string client_user_agent, std::string remote_endpoint) - : logger_ptr{std::make_shared(make_logger_prefix(id), serv.logger_ptr)} // Throws - , logger{*logger_ptr} + : logger{make_logger_prefix(id), serv.logger} // Throws , m_server{serv} , m_id{id} , m_socket{std::move(socket)} @@ -1130,9 +1128,9 @@ class SyncConnection : public websocket::Config { return m_remote_endpoint; } - const std::shared_ptr& websocket_get_logger() noexcept final + util::Logger& websocket_get_logger() noexcept final { - return logger_ptr; + return logger; } std::mt19937_64& websocket_get_random() noexcept final override @@ -1438,17 +1436,15 @@ std::string g_user_agent = "User-Agent"; class HTTPConnection { public: - const std::shared_ptr logger_ptr; - util::Logger& logger; + util::PrefixLogger logger; HTTPConnection(ServerImpl& serv, int_fast64_t id, bool is_ssl) - : logger_ptr{std::make_shared(make_logger_prefix(id), serv.logger_ptr)} // Throws - , logger{*logger_ptr} + : logger{make_logger_prefix(id), serv.logger} // Throws , m_server{serv} , m_id{id} , m_socket{new network::Socket{serv.get_service()}} // Throws , m_read_ahead_buffer{new network::ReadAheadBuffer} // Throws - , m_http_server{*this, logger_ptr} + , m_http_server{*this, logger} { // Make the output buffer stream throw std::bad_alloc if it fails to // expand the buffer @@ -2052,7 +2048,7 @@ class Session final : private FileIdentReceiver { util::PrefixLogger logger; Session(SyncConnection& conn, session_ident_type session_ident) - : logger{make_logger_prefix(session_ident), conn.logger_ptr} // Throws + : logger{make_logger_prefix(session_ident), conn.logger} // Throws , m_connection{conn} , m_session_ident{session_ident} { @@ -3160,7 +3156,7 @@ void SessionQueue::clear() noexcept ServerFile::ServerFile(ServerImpl& server, ServerFileAccessCache& cache, const std::string& virt_path, std::string real_path, bool disable_sync_to_disk) - : logger{"ServerFile[" + virt_path + "]: ", server.logger_ptr} // Throws + : logger{"ServerFile[" + virt_path + "]: ", server.logger} // Throws , wlogger{"ServerFile[" + virt_path + "]: ", server.get_worker().logger} // Throws , m_server{server} , m_file{cache, real_path, virt_path, false, disable_sync_to_disk} // Throws @@ -3696,7 +3692,7 @@ void ServerFile::finalize_work_stage_2() // ============================ Worker implementation ============================ Worker::Worker(ServerImpl& server) - : logger{"Worker: ", server.logger_ptr} // Throws + : logger{"Worker: ", server.logger} // Throws , m_server{server} , m_transformer{make_transformer()} // Throws , m_file_access_cache{server.get_config().max_open_files, logger, *this, server.get_config().encryption_key} diff --git a/src/realm/sync/socket_provider.hpp b/src/realm/sync/socket_provider.hpp index 265143c0331..2a7b93e812d 100644 --- a/src/realm/sync/socket_provider.hpp +++ b/src/realm/sync/socket_provider.hpp @@ -97,7 +97,7 @@ class SyncSocketProvider { /// websocket will call directly to the handlers provided by the observer. /// The WebSocketObserver guarantees that the WebSocket object will be /// closed/destroyed before the observer is terminated/destroyed. - virtual std::unique_ptr connect(std::unique_ptr observer, + virtual std::unique_ptr connect(WebSocketObserver* observer, WebSocketEndpoint&& endpoint) = 0; /// Submit a handler function to be executed by the event loop (thread). @@ -142,7 +142,8 @@ class SyncSocketProvider { /// Temporary functions added to support the default socket provider until /// it is fully integrated. Will be removed in future PRs. - virtual void stop(bool = false) {} + virtual void run() {} + virtual void stop() {} }; /// Struct that defines the endpoint to create a new websocket connection. @@ -247,6 +248,17 @@ struct WebSocketObserver { /// is returned, the WebSocket object will be destroyed at some point /// in the future. virtual bool websocket_closed_handler(bool was_clean, Status status) = 0; + + //@{ + /// DEPRECATED - Will be removed in a future release + /// These functions are deprecated and should not be called by custom socket provider implementations + virtual void websocket_connect_error_handler(std::error_code) = 0; + virtual void websocket_ssl_handshake_error_handler(std::error_code) = 0; + virtual void websocket_read_or_write_error_handler(std::error_code) = 0; + virtual void websocket_handshake_error_handler(std::error_code, const std::string_view* body) = 0; + virtual void websocket_protocol_error_handler(std::error_code) = 0; + virtual bool websocket_close_message_received(std::error_code error_code, StringData message) = 0; + //@} }; } // namespace realm::sync diff --git a/src/realm/sync/tools/apply_to_state_command.cpp b/src/realm/sync/tools/apply_to_state_command.cpp index 99a93e4929f..99cdc375f1c 100644 --- a/src/realm/sync/tools/apply_to_state_command.cpp +++ b/src/realm/sync/tools/apply_to_state_command.cpp @@ -286,11 +286,10 @@ int main(int argc, const char** argv) mpark::visit(realm::util::overload{ [&](const DownloadMessage& download_message) { realm::sync::VersionInfo version_info; - auto transact = bool(flx_sync_arg) ? local_db->start_write() : local_db->start_read(); history.integrate_server_changesets(download_message.progress, &download_message.downloadable_bytes, download_message.changesets, version_info, - download_message.batch_state, *logger, transact); + download_message.batch_state, *logger, nullptr); }, [&](const UploadMessage& upload_message) { for (const auto& changeset : upload_message.changesets) { diff --git a/src/realm/sync/transform.cpp b/src/realm/sync/transform.cpp index 209e1ba1d4e..8ac67259022 100644 --- a/src/realm/sync/transform.cpp +++ b/src/realm/sync/transform.cpp @@ -52,6 +52,9 @@ namespace { #endif #endif +#define REALM_MERGE_ASSERT(condition) \ + (REALM_LIKELY(condition) ? static_cast(0) : throw sync::TransformError{"Assertion failed: " #condition}) + } // unnamed namespace using namespace realm; @@ -892,25 +895,6 @@ void TransformerImpl::MinorSide::prepend(InputIterator begin, InputIterator end) namespace { -REALM_NORETURN void throw_bad_merge(std::string msg) -{ - throw sync::TransformError{std::move(msg)}; -} - -template -REALM_NORETURN void bad_merge(const char* msg, Params&&... params) -{ - throw_bad_merge(util::format(msg, std::forward(params)...)); -} - -REALM_NORETURN void bad_merge(_impl::TransformerImpl::Side& side, Instruction::PathInstruction instr, - const std::string& msg) -{ - std::stringstream ss; - side.m_changeset->print_path(ss, instr.table, instr.object, instr.field, &instr.path); - bad_merge("%1 (instruction target: %2). Please contact support", msg, ss.str()); -} - template struct Merge; template @@ -991,7 +975,7 @@ struct MergeUtils { return left.data.uuid == right.data.uuid; } - bad_merge("Invalid payload type in instruction"); + REALM_MERGE_ASSERT(false && "Invalid payload type in instruction"); } bool same_path_element(const Instruction::Path::Element& left, @@ -1226,7 +1210,7 @@ struct MergeUtils { REALM_ASSERT(mpark::holds_alternative(left.path.back())); size_t index = left.path.size() - 1; if (!mpark::holds_alternative(right.path[index])) { - bad_merge("Inconsistent paths"); + throw TransformError{"Inconsistent paths"}; } return mpark::get(right.path[index]); } @@ -1405,35 +1389,44 @@ DEFINE_MERGE(Instruction::AddTable, Instruction::AddTable) StringData left_pk_name = left_side.get_string(left_spec->pk_field); StringData right_pk_name = right_side.get_string(right_spec->pk_field); if (left_pk_name != right_pk_name) { - bad_merge( - "Schema mismatch: '%1' has primary key '%2' on one side, but primary key '%3' on the other.", - left_name, left_pk_name, right_pk_name); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' has primary key '" << left_pk_name + << "' on one side, but primary key '" << right_pk_name << "' on the other."; + throw SchemaMismatchError(ss.str()); } if (left_spec->pk_type != right_spec->pk_type) { - bad_merge("Schema mismatch: '%1' has primary key '%2', which is of type %3 on one side and type " - "%4 on the other.", - left_name, left_pk_name, get_type_name(left_spec->pk_type), - get_type_name(right_spec->pk_type)); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' has primary key '" << left_pk_name + << "', which is of type " << get_type_name(left_spec->pk_type) << " on one side and type " + << get_type_name(right_spec->pk_type) << " on the other."; + throw SchemaMismatchError(ss.str()); } if (left_spec->pk_nullable != right_spec->pk_nullable) { - bad_merge("Schema mismatch: '%1' has primary key '%2', which is nullable on one side, but not " - "the other.", - left_name, left_pk_name); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' has primary key '" << left_pk_name + << "', which is nullable on one side, but not the other."; + throw SchemaMismatchError(ss.str()); } if (left_spec->is_asymmetric != right_spec->is_asymmetric) { - bad_merge("Schema mismatch: '%1' is asymmetric on one side, but not on the other.", left_name); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' is asymmetric on one side, but not on the other."; + throw SchemaMismatchError(ss.str()); } } else { - bad_merge("Schema mismatch: '%1' has a primary key on one side, but not on the other.", left_name); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' has a primary key on one side, but not on the other."; + throw SchemaMismatchError(ss.str()); } } else if (mpark::get_if(&left.type)) { if (!mpark::get_if(&right.type)) { - bad_merge("Schema mismatch: '%1' is an embedded table on one side, but not the other.", left_name); + std::stringstream ss; + ss << "Schema mismatch: '" << left_name << "' is an embedded table on one side, but not the other."; + throw SchemaMismatchError(ss.str()); } } @@ -1608,19 +1601,15 @@ DEFINE_MERGE(Instruction::Update, Instruction::Update) if (same_path(left, right)) { bool left_is_default = false; bool right_is_default = false; - if (!(left.is_array_update() == right.is_array_update())) { - bad_merge(left_side, left, "Merge error: left.is_array_update() == right.is_array_update()"); - } + REALM_MERGE_ASSERT(left.is_array_update() == right.is_array_update()); if (!left.is_array_update()) { - if (right.is_array_update()) { - bad_merge(right_side, right, "Merge error: !right.is_array_update()"); - } + REALM_MERGE_ASSERT(!right.is_array_update()); left_is_default = left.is_default; right_is_default = right.is_default; } - else if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); + else { + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); } if (left.value.type != right.value.type) { @@ -1672,10 +1661,7 @@ DEFINE_MERGE(Instruction::AddInteger, Instruction::Update) // RESOLUTION: If the Add was later than the Set, add its value to // the payload of the Set instruction. Otherwise, discard it. - if (!(right.value.type == Instruction::Payload::Type::Int || right.value.is_null())) { - bad_merge(right_side, right, - "Merge error: right.value.type == Instruction::Payload::Type::Int || right.value.is_null()"); - } + REALM_MERGE_ASSERT(right.value.type == Instruction::Payload::Type::Int || right.value.is_null()); bool right_is_default = !right.is_array_update() && right.is_default; @@ -1712,15 +1698,9 @@ DEFINE_MERGE(Instruction::ArrayInsert, Instruction::Update) { if (same_container(left, right)) { REALM_ASSERT(right.is_array_update()); - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() <= left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() <= left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() <= left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); right.prior_size += 1; if (right.index() >= left.index()) { right.index() += 1; // ---> @@ -1733,12 +1713,8 @@ DEFINE_MERGE(Instruction::ArrayMove, Instruction::Update) if (same_container(left, right)) { REALM_ASSERT(right.is_array_update()); - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); // FIXME: This marks both sides as dirty, even when they are unmodified. merge_get_vs_move(right.index(), left.index(), left.ndx_2); @@ -1749,15 +1725,9 @@ DEFINE_MERGE(Instruction::ArrayErase, Instruction::Update) { if (same_container(left, right)) { REALM_ASSERT(right.is_array_update()); - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); // CONFLICT: Update of a removed element. // @@ -1805,14 +1775,18 @@ DEFINE_MERGE(Instruction::AddColumn, Instruction::AddColumn) if (same_column(left, right)) { StringData left_name = left_side.get_string(left.field); if (left.type != right.type) { - bad_merge( - "Schema mismatch: Property '%1' in class '%2' is of type %3 on one side and type %4 on the other.", - left_name, left_side.get_string(left.table), get_type_name(left.type), get_type_name(right.type)); + std::stringstream ss; + ss << "Schema mismatch: Property '" << left_name << "' in class '" << left_side.get_string(left.table) + << "' is of type " << get_type_name(left.type) << " on one side and type " << get_type_name(right.type) + << " on the other."; + throw SchemaMismatchError(ss.str()); } if (left.nullable != right.nullable) { - bad_merge("Schema mismatch: Property '%1' in class '%2' is nullable on one side and not on the other.", - left_name, left_side.get_string(left.table)); + std::stringstream ss; + ss << "Schema mismatch: Property '" << left_name << "' in class '" << left_side.get_string(left.table) + << "' is nullable on one side and not on the other."; + throw SchemaMismatchError(ss.str()); } if (left.collection_type != right.collection_type) { @@ -1833,17 +1807,20 @@ DEFINE_MERGE(Instruction::AddColumn, Instruction::AddColumn) std::stringstream ss; const char* left_type = collection_type_name(left.collection_type); const char* right_type = collection_type_name(right.collection_type); - bad_merge("Schema mismatch: Property '%1' in class '%2' is a %3 on one side, and a %4 on the other.", - left_name, left_side.get_string(left.table), left_type, right_type); + ss << "Schema mismatch: Property '" << left_name << "' in class '" << left_side.get_string(left.table) + << "' is a " << left_type << " on one side, and a " << right_type << " on the other."; + throw SchemaMismatchError(ss.str()); } if (left.type == Instruction::Payload::Type::Link) { StringData left_target = left_side.get_string(left.link_target_table); StringData right_target = right_side.get_string(right.link_target_table); if (left_target != right_target) { - bad_merge("Schema mismatch: Link property '%1' in class '%2' points to class '%3' on one side and to " - "'%4' on the other.", - left_name, left_side.get_string(left.table), left_target, right_target); + std::stringstream ss; + ss << "Schema mismatch: Link property '" << left_name << "' in class '" + << left_side.get_string(left.table) << "' points to class '" << left_target + << "' on one side and to '" << right_target << "' on the other."; + throw SchemaMismatchError(ss.str()); } } @@ -1903,9 +1880,7 @@ DEFINE_NESTED_MERGE(Instruction::ArrayInsert) DEFINE_MERGE(Instruction::ArrayInsert, Instruction::ArrayInsert) { if (same_container(left, right)) { - if (!(left.prior_size == right.prior_size)) { - bad_merge(right_side, right, "Exception: left.prior_size == right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); left.prior_size++; right.prior_size++; @@ -1968,15 +1943,9 @@ DEFINE_MERGE(Instruction::ArrayMove, Instruction::ArrayInsert) DEFINE_MERGE(Instruction::ArrayErase, Instruction::ArrayInsert) { if (same_container(left, right)) { - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() <= right.prior_size)) { - bad_merge(left_side, left, "Merge error: right.index() <= right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() <= right.prior_size); left.prior_size++; right.prior_size--; @@ -2008,21 +1977,11 @@ DEFINE_NESTED_MERGE(Instruction::ArrayMove) DEFINE_MERGE(Instruction::ArrayMove, Instruction::ArrayMove) { if (same_container(left, right)) { - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } - if (!(left.ndx_2 < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.ndx_2 < left.prior_size"); - } - if (!(right.ndx_2 < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.ndx_2 < right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); + REALM_MERGE_ASSERT(left.ndx_2 < left.prior_size); + REALM_MERGE_ASSERT(right.ndx_2 < right.prior_size); if (left.index() < right.index()) { right.index() -= 1; // <--- @@ -2103,15 +2062,9 @@ DEFINE_MERGE(Instruction::ArrayMove, Instruction::ArrayMove) DEFINE_MERGE(Instruction::ArrayErase, Instruction::ArrayMove) { if (same_container(left, right)) { - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); right.prior_size -= 1; @@ -2176,15 +2129,9 @@ DEFINE_NESTED_MERGE(Instruction::ArrayErase) DEFINE_MERGE(Instruction::ArrayErase, Instruction::ArrayErase) { if (same_container(left, right)) { - if (!(left.prior_size == right.prior_size)) { - bad_merge(left_side, left, "Merge error: left.prior_size == right.prior_size"); - } - if (!(left.index() < left.prior_size)) { - bad_merge(left_side, left, "Merge error: left.index() < left.prior_size"); - } - if (!(right.index() < right.prior_size)) { - bad_merge(right_side, right, "Merge error: right.index() < right.prior_size"); - } + REALM_MERGE_ASSERT(left.prior_size == right.prior_size); + REALM_MERGE_ASSERT(left.index() < left.prior_size); + REALM_MERGE_ASSERT(right.index() < right.prior_size); left.prior_size -= 1; right.prior_size -= 1; diff --git a/src/realm/sync/transform.hpp b/src/realm/sync/transform.hpp index f772e390d36..d9cb4d72a0c 100644 --- a/src/realm/sync/transform.hpp +++ b/src/realm/sync/transform.hpp @@ -296,6 +296,11 @@ class TransformError : public std::runtime_error { } }; +class SchemaMismatchError : public TransformError { +public: + using TransformError::TransformError; +}; + inline Transformer::RemoteChangeset::RemoteChangeset(version_type rv, version_type lv, ChunkedBinaryData d, timestamp_type ot, file_ident_type fi) : remote_version(rv) diff --git a/src/realm/transaction.cpp b/src/realm/transaction.cpp index c50c3033e15..26bc5e46ce6 100644 --- a/src/realm/transaction.cpp +++ b/src/realm/transaction.cpp @@ -272,7 +272,7 @@ VersionID Transaction::commit_and_continue_as_read(bool commit_to_disk) } } -VersionID Transaction::commit_and_continue_writing() +void Transaction::commit_and_continue_writing() { if (!is_attached()) throw LogicError(LogicError::wrong_transact_state); @@ -284,7 +284,7 @@ VersionID Transaction::commit_and_continue_writing() // before committing, allow any accessors at group level or below to sync flush_accessors_for_commit(); - DB::version_type version = db->do_commit(*this); // Throws + db->do_commit(*this); // Throws // We need to set m_read_lock in order for wait_for_change to work. // To set it, we grab a readlock on the latest available snapshot @@ -299,7 +299,6 @@ VersionID Transaction::commit_and_continue_writing() bool writable = true; remap_and_update_refs(m_read_lock.m_top_ref, m_read_lock.m_file_size, writable); // Throws - return VersionID{version, lock_after_commit.m_reader_idx}; } TransactionRef Transaction::freeze() @@ -814,9 +813,9 @@ void Transaction::set_transact_stage(DB::TransactStage stage) noexcept class NodeTree { public: - NodeTree(size_t evac_limit, size_t work_limit) + NodeTree(size_t evac_limit, size_t& work_limit) : m_evac_limit(evac_limit) - , m_work_limit(int64_t(work_limit)) + , m_work_limit(work_limit) , m_moved(0) { } @@ -836,15 +835,18 @@ class NodeTree { /// to point to the node we have just processed bool trv(Array& current_node, unsigned level, std::vector& progress) { - if (m_work_limit < 0) { - return false; - } if (current_node.is_read_only()) { - size_t byte_size = current_node.get_byte_size(); + auto byte_size = current_node.get_byte_size(); if ((current_node.get_ref() + byte_size) > m_evac_limit) { current_node.copy_on_write(); m_moved++; - m_work_limit -= byte_size; + if (m_work_limit > byte_size) { + m_work_limit -= byte_size; + } + else { + m_work_limit = 0; + return false; + } } } @@ -876,12 +878,12 @@ class NodeTree { private: size_t m_evac_limit; - int64_t m_work_limit; + size_t m_work_limit; size_t m_moved; }; -void Transaction::cow_outliers(std::vector& progress, size_t evac_limit, size_t work_limit) +void Transaction::cow_outliers(std::vector& progress, size_t evac_limit, size_t& work_limit) { NodeTree node_tree(evac_limit, work_limit); if (progress.empty()) { diff --git a/src/realm/transaction.hpp b/src/realm/transaction.hpp index 119babc2a8e..be95738a321 100644 --- a/src/realm/transaction.hpp +++ b/src/realm/transaction.hpp @@ -60,7 +60,7 @@ class Transaction : public Group { // Live transactions state changes, often taking an observer functor: VersionID commit_and_continue_as_read(bool commit_to_disk = true) REQUIRES(!m_async_mutex); - VersionID commit_and_continue_writing(); + void commit_and_continue_writing(); template void rollback_and_continue_as_read(O* observer) REQUIRES(!m_async_mutex); void rollback_and_continue_as_read() REQUIRES(!m_async_mutex) @@ -188,7 +188,7 @@ class Transaction : public Group { void complete_async_commit(); void acquire_write_lock() REQUIRES(!m_async_mutex); - void cow_outliers(std::vector& progress, size_t evac_limit, size_t work_limit); + void cow_outliers(std::vector& progress, size_t evac_limit, size_t& work_limit); void close_read_with_lock() REQUIRES(!m_async_mutex, db->m_mutex); DBRef db; diff --git a/src/realm/util/basic_system_errors.cpp b/src/realm/util/basic_system_errors.cpp index e706787e88c..7c0d45b0824 100644 --- a/src/realm/util/basic_system_errors.cpp +++ b/src/realm/util/basic_system_errors.cpp @@ -33,6 +33,8 @@ class system_category : public std::error_category { std::string message(int) const override; }; +system_category g_system_category; + const char* system_category::name() const noexcept { return "realm.basic_system"; @@ -98,13 +100,7 @@ namespace error { std::error_code make_error_code(basic_system_errors err) noexcept { - return std::error_code(err, basic_system_error_category()); -} - -const std::error_category& basic_system_error_category() -{ - static system_category system_category; - return system_category; + return std::error_code(err, g_system_category); } } // namespace error diff --git a/src/realm/util/basic_system_errors.hpp b/src/realm/util/basic_system_errors.hpp index be261fed47f..8f7a626eafb 100644 --- a/src/realm/util/basic_system_errors.hpp +++ b/src/realm/util/basic_system_errors.hpp @@ -54,7 +54,6 @@ enum basic_system_errors { }; std::error_code make_error_code(basic_system_errors) noexcept; -const std::error_category& basic_system_error_category(); } // namespace error } // namespace util diff --git a/src/realm/util/buffer_stream.hpp b/src/realm/util/buffer_stream.hpp index ac9eca44899..cb51ab6ef01 100644 --- a/src/realm/util/buffer_stream.hpp +++ b/src/realm/util/buffer_stream.hpp @@ -22,8 +22,6 @@ #include #include -#include - namespace realm { namespace util { @@ -49,9 +47,6 @@ class BasicResettableExpandableOutputStreambuf : public std::basic_stringbuf as_span() noexcept; - util::Span as_span() const noexcept; }; @@ -74,9 +69,6 @@ class BasicResettableExpandableBufferOutputStream : public std::basic_ostream as_span() noexcept; - util::Span as_span() const noexcept; - private: BasicResettableExpandableOutputStreambuf m_streambuf; }; @@ -116,18 +108,6 @@ inline std::size_t BasicResettableExpandableOutputStreambuf::size() con return size; } -template -inline util::Span BasicResettableExpandableOutputStreambuf::as_span() noexcept -{ - return util::Span(data(), size()); -} - -template -inline util::Span BasicResettableExpandableOutputStreambuf::as_span() const noexcept -{ - return util::Span(data(), size()); -} - template inline BasicResettableExpandableBufferOutputStream::BasicResettableExpandableBufferOutputStream() : std::basic_ostream(&m_streambuf) // Throws @@ -160,18 +140,6 @@ inline std::size_t BasicResettableExpandableBufferOutputStream::size() return m_streambuf.size(); } -template -inline util::Span BasicResettableExpandableBufferOutputStream::as_span() noexcept -{ - return util::Span(m_streambuf.data(), m_streambuf.size()); -} - -template -inline util::Span BasicResettableExpandableBufferOutputStream::as_span() const noexcept -{ - return util::Span(m_streambuf.data(), m_streambuf.size()); -} - } // namespace util } // namespace realm diff --git a/src/realm/util/encrypted_file_mapping.cpp b/src/realm/util/encrypted_file_mapping.cpp index 8a2fe18bb0d..7815e4bfb86 100644 --- a/src/realm/util/encrypted_file_mapping.cpp +++ b/src/realm/util/encrypted_file_mapping.cpp @@ -820,8 +820,7 @@ void EncryptedFileMapping::write_barrier(const void* addr, size_t size) noexcept // propagate changes to first page (update may be partial, may also be to last page) if (first_accessed_local_page < pages_size) { - REALM_ASSERT_EX(is(m_page_state[first_accessed_local_page], UpToDate), - m_page_state[first_accessed_local_page]); + REALM_ASSERT(is(m_page_state[first_accessed_local_page], UpToDate)); if (first_accessed_local_page == last_accessed_local_page) { size_t last_offset = last_accessed_address - page_addr(first_accessed_local_page); write_and_update_all(first_accessed_local_page, first_offset, last_offset + 1); diff --git a/src/realm/util/features.h b/src/realm/util/features.h index 7bb1bc4c8c6..c05fcdad57e 100644 --- a/src/realm/util/features.h +++ b/src/realm/util/features.h @@ -124,6 +124,18 @@ #define REALM_DIAG_IGNORE_UNSIGNED_MINUS() #endif +/* Compiler is MSVC (Microsoft Visual C++) */ +#if defined(_MSC_VER) && _MSC_VER >= 1600 +#define REALM_HAVE_AT_LEAST_MSVC_10_2010 1 +#endif +#if defined(_MSC_VER) && _MSC_VER >= 1700 +#define REALM_HAVE_AT_LEAST_MSVC_11_2012 1 +#endif +#if defined(_MSC_VER) && _MSC_VER >= 1800 +#define REALM_HAVE_AT_LEAST_MSVC_12_2013 1 +#endif + + /* The way to specify that a function never returns. */ #if REALM_HAVE_AT_LEAST_GCC(4, 8) || REALM_HAVE_CLANG_FEATURE(cxx_attributes) #define REALM_NORETURN [[noreturn]] @@ -240,10 +252,8 @@ /* Device (iPhone or iPad) or simulator. */ #define REALM_IOS 1 #define REALM_APPLE_DEVICE !TARGET_OS_SIMULATOR -#define REALM_MACCATALYST TARGET_OS_MACCATALYST #else #define REALM_IOS 0 -#define REALM_MACCATALYST 0 #endif #if TARGET_OS_WATCH == 1 /* Device (Apple Watch) or simulator. */ @@ -261,7 +271,6 @@ #endif #else #define REALM_PLATFORM_APPLE 0 -#define REALM_MACCATALYST 0 #define REALM_IOS 0 #define REALM_WATCHOS 0 #define REALM_TVOS 0 @@ -296,18 +305,6 @@ #define REALM_ARCHITECTURE_X86_64 0 #endif -#if defined(__arm__) || defined(_M_ARM) -#define REALM_ARCHITECTURE_ARM32 1 -#else -#define REALM_ARCHITECTURE_ARM32 0 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) -#define REALM_ARCHITECTURE_ARM64 1 -#else -#define REALM_ARCHITECTURE_ARM64 0 -#endif - // Address Sanitizer #if defined(__has_feature) // Clang # if __has_feature(address_sanitizer) diff --git a/src/realm/util/file.cpp b/src/realm/util/file.cpp index 3c0ae0196bb..a7df088f038 100644 --- a/src/realm/util/file.cpp +++ b/src/realm/util/file.cpp @@ -29,7 +29,11 @@ #include #include -#ifndef _WIN32 +#ifdef _WIN32 +#include +#include +#include +#else #include #include #include @@ -49,6 +53,45 @@ using namespace realm; using namespace realm::util; namespace { +#ifdef _WIN32 // Windows - GetLastError() + +#undef max + +std::string get_last_error_msg(const char* prefix, DWORD err) +{ + std::string buffer; + buffer.append(prefix); + size_t offset = buffer.size(); + size_t max_msg_size = 1024; + buffer.resize(offset + max_msg_size); + DWORD flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + DWORD language_id = MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT); + DWORD size = + FormatMessageA(flags, 0, err, language_id, buffer.data() + offset, static_cast(max_msg_size), 0); + if (0 < size) + return buffer; + buffer.resize(offset); + buffer.append("Unknown error"); + return buffer; +} + +std::wstring string_to_wstring(const std::string& str) +{ + if (str.empty()) + return std::wstring(); + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, NULL, 0); + std::wstring wstr; + if (wstr_size) { + wstr.resize(wstr_size); + if (MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, &wstr[0], wstr_size)) { + return wstr; + } + } + return std::wstring(); +} + +#endif + size_t get_page_size() { #ifdef _WIN32 @@ -90,22 +133,23 @@ bool for_each_helper(const std::string& path, const std::string& dir, File::ForE #ifdef _WIN32 -std::string get_last_error_msg(const char* prefix, DWORD err) +std::chrono::system_clock::time_point file_time_to_system_clock(FILETIME ft) { - std::string buffer; - buffer.append(prefix); - size_t offset = buffer.size(); - size_t max_msg_size = 1024; - buffer.resize(offset + max_msg_size); - DWORD flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; - DWORD language_id = MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT); - DWORD size = - FormatMessageA(flags, 0, err, language_id, buffer.data() + offset, static_cast(max_msg_size), 0); - if (0 < size) - return buffer; - buffer.resize(offset); - buffer.append("Unknown error"); - return buffer; + // Microseconds between 1601-01-01 00:00:00 UTC and 1970-01-01 00:00:00 UTC + constexpr uint64_t kEpochDifferenceMicros = 11644473600000000ull; + + // Construct a 64 bit value that is the number of nanoseconds from the + // Windows epoch which is 1601-01-01 00:00:00 UTC + auto totalMicros = static_cast(ft.dwHighDateTime) << 32; + totalMicros |= static_cast(ft.dwLowDateTime); + + // FILETIME is 100's of nanoseconds since Windows epoch + totalMicros /= 10; + // Move it from micros since the Windows epoch to micros since the Unix epoch + totalMicros -= kEpochDifferenceMicros; + + std::chrono::duration totalMicrosDur(totalMicros); + return std::chrono::system_clock::time_point(totalMicrosDur); } struct WindowsFileHandleHolder { @@ -137,39 +181,6 @@ struct WindowsFileHandleHolder { #endif -#if REALM_HAVE_STD_FILESYSTEM -void throwIfCreateDirectoryError(std::error_code error, const std::string& path) -{ - if (!error) - return; - - // create_directory doesn't raise an error if the path already exists - using std::errc; - if (error == errc::permission_denied || error == errc::read_only_file_system) { - throw File::PermissionDenied(error.message(), path); - } - else { - throw File::AccessError(error.message(), path); - } -} - -void throwIfFileError(std::error_code error, const std::string& path) -{ - if (!error) - return; - - using std::errc; - if (error == errc::permission_denied || error == errc::read_only_file_system || - error == errc::device_or_resource_busy || error == errc::operation_not_permitted || - error == errc::file_exists || error == errc::directory_not_empty) { - throw File::PermissionDenied(error.message(), path); - } - else { - throw File::AccessError(error.message(), path); - } -} -#endif - } // anonymous namespace @@ -179,18 +190,16 @@ namespace util { bool try_make_dir(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - bool result = std::filesystem::create_directory(path, error); - throwIfCreateDirectoryError(error, path); - return result; +#ifdef _WIN32 + std::wstring w_path = string_to_wstring(path); + if (_wmkdir(w_path.c_str()) == 0) + return true; #else // POSIX if (::mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) == 0) return true; - +#endif int err = errno; // Eliminate any risk of clobbering std::string msg = get_errno_msg("make_dir() failed: ", err); - switch (err) { case EEXIST: return false; @@ -200,7 +209,6 @@ bool try_make_dir(const std::string& path) default: throw File::AccessError(msg, path); // LCOV_EXCL_LINE } -#endif } @@ -215,11 +223,6 @@ void make_dir(const std::string& path) void make_dir_recursive(std::string path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - std::filesystem::create_directories(path, error); - throwIfCreateDirectoryError(error, path); -#else // Skip the first separator as we're assuming an absolute path size_t pos = path.find_first_of("/\\"); if (pos == std::string::npos) @@ -238,7 +241,6 @@ void make_dir_recursive(std::string path) path[sep] = c; pos = sep + 1; } -#endif } @@ -254,15 +256,14 @@ void remove_dir(const std::string& path) bool try_remove_dir(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - bool result = std::filesystem::remove(path, error); - throwIfFileError(error, path); - return result; +#ifdef _WIN32 + std::wstring w_path = string_to_wstring(path); + if (_wrmdir(w_path.c_str()) == 0) + return true; #else // POSIX if (::rmdir(path.c_str()) == 0) return true; - +#endif int err = errno; // Eliminate any risk of clobbering std::string msg = get_errno_msg("remove_dir() failed: ", err); switch (err) { @@ -278,7 +279,6 @@ bool try_remove_dir(const std::string& path) default: throw File::AccessError(msg, path); // LCOV_EXCL_LINE } -#endif } @@ -295,12 +295,6 @@ void remove_dir_recursive(const std::string& path) bool try_remove_dir_recursive(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - auto removed_count = std::filesystem::remove_all(path, error); - throwIfFileError(error, path); - return removed_count > 0; -#else { bool allow_missing = true; DirScanner ds{path, allow_missing}; // Throws @@ -316,7 +310,6 @@ bool try_remove_dir_recursive(const std::string& path) } } return try_remove_dir(path); // Throws -#endif } @@ -416,8 +409,8 @@ void File::open_internal(const std::string& path, AccessMode a, CreateMode c, in break; } DWORD flags_and_attributes = 0; - HANDLE handle = - CreateFile2(std::filesystem::path(path).c_str(), desired_access, share_mode, creation_disposition, nullptr); + std::wstring ws = string_to_wstring(path); + HANDLE handle = CreateFile2(ws.c_str(), desired_access, share_mode, creation_disposition, nullptr); if (handle != INVALID_HANDLE_VALUE) { m_fd = handle; m_have_lock = false; @@ -987,7 +980,11 @@ bool File::is_prealloc_supported() void File::seek(SizeType position) { REALM_ASSERT_RELEASE(is_attached()); +#ifdef _WIN32 + seek_static(m_fd, position); +#else seek_static(m_fd, position); +#endif } void File::seek_static(FileDesc fd, SizeType position) @@ -1066,15 +1063,41 @@ static void _unlock(int m_fd) } #endif -bool File::rw_lock(bool exclusive, bool non_blocking) +bool File::lock(bool exclusive, bool non_blocking) { - // exclusive blocking rw locks not implemented for emulation - REALM_ASSERT(!exclusive || non_blocking); + REALM_ASSERT_RELEASE(is_attached()); + +#ifdef _WIN32 // Windows version + + REALM_ASSERT_RELEASE(!m_have_lock); + + // Under Windows a file lock must be explicitely released before + // the file is closed. It will eventually be released by the + // system, but there is no guarantees on the timing. + + DWORD flags = 0; + if (exclusive) + flags |= LOCKFILE_EXCLUSIVE_LOCK; + if (non_blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + OVERLAPPED overlapped; + memset(&overlapped, 0, sizeof overlapped); + overlapped.Offset = 0; // Just for clarity + overlapped.OffsetHigh = 0; // Just for clarity + if (LockFileEx(m_fd, flags, 0, 1, 0, &overlapped)) { + m_have_lock = true; + return true; + } + DWORD err = GetLastError(); // Eliminate any risk of clobbering + if (err == ERROR_LOCK_VIOLATION) + return false; + throw std::system_error(err, std::system_category(), "LockFileEx() failed"); -#ifndef REALM_FILELOCK_EMULATION - return lock(exclusive, non_blocking); #else - REALM_ASSERT(!m_has_exclusive_lock && !has_shared_lock()); +#ifdef REALM_FILELOCK_EMULATION + // If we already have any form of lock, release it before trying to acquire a new + if (m_has_exclusive_lock || has_shared_lock()) + unlock(); // First obtain an exclusive lock on the file proper int operation = LOCK_EX; @@ -1090,10 +1113,6 @@ bool File::rw_lock(bool exclusive, bool non_blocking) throw std::system_error(errno, std::system_category(), "flock() failed"); m_has_exclusive_lock = true; - // Every path through this function except for successfully acquiring an - // exclusive lock needs to release the flock() before returning. - UnlockGuard ulg(*this); - // now use a named pipe to emulate locking in conjunction with using exclusive lock // on the file proper. // exclusively locked: we can't sucessfully write to the pipe. @@ -1105,94 +1124,72 @@ bool File::rw_lock(bool exclusive, bool non_blocking) REALM_ASSERT_RELEASE(m_pipe_fd == -1); if (m_fifo_path.empty()) m_fifo_path = m_path + ".fifo"; - - // Due to a bug in iOS 10-12 we need to open in read-write mode for shared - // or the OS will deadlock when un-suspending the app. - int mode = exclusive ? O_WRONLY | O_NONBLOCK : O_RDWR | O_NONBLOCK; - - // Optimistically try to open the fifo. This may fail due to the fifo not - // existing, but since it usually exists this is faster than trying to create - // it first. - int fd = ::open(m_fifo_path.c_str(), mode); - if (fd == -1) { + status = mkfifo(m_fifo_path.c_str(), 0666); + if (status) { int err = errno; - if (exclusive) { - if (err == ENOENT || err == ENXIO) { - // If the fifo either doesn't exist or there's no readers with the - // other end of the pipe open (ENXIO) then we have an exclusive lock - // and are done. - ulg.release(); + REALM_ASSERT_EX(status == -1, status); + if (err == ENOENT) { + // The management directory doesn't exist, so there's clearly no + // readers. This can happen when calling DB::call_with_lock() or + // if the management directory has been removed by DB::call_with_lock() + if (exclusive) { return true; } - - // Otherwise we got an unexpected error - throw std::system_error(err, std::system_category(), "opening lock fifo for writing failed"); - } - - if (err == ENOENT) { - // The fifo doesn't exist and we're opening in shared mode, so we - // need to create it. + // open shared: + // We need the fifo in order to make a shared lock. If we have it + // in a management directory, we may need to create that first: if (!m_fifo_dir_path.empty()) try_make_dir(m_fifo_dir_path); + // now we can try creating the FIFO again status = mkfifo(m_fifo_path.c_str(), 0666); - if (status != 0) - throw std::system_error(errno, std::system_category(), "creating lock fifo for reading failed"); - - // Try again to open the fifo now that it exists - fd = ::open(m_fifo_path.c_str(), mode); - err = errno; + if (status) { + // If we fail it must be because it already exists + err = errno; + REALM_ASSERT_EX(err == EEXIST, err); + } + } + else { + // if we failed to create the fifo and not because dir is missing, + // it must be because the fifo already exists! + REALM_ASSERT_EX(err == EEXIST, err); } - - if (fd == -1) - throw std::system_error(err, std::system_category(), "opening lock fifo for reading failed"); } - - // We successfully opened the pipe. If we're trying to acquire an exclusive - // lock that means there's a reader (aka a shared lock) and we've failed. - // Release the exclusive lock and back out. if (exclusive) { - ::close(fd); - return false; + // check if any shared locks are already taken by trying to open the pipe for writing + // shared locks are indicated by one or more readers already having opened the pipe + int fd; + do { + fd = ::open(m_fifo_path.c_str(), O_WRONLY | O_NONBLOCK); + } while (fd == -1 && errno == EINTR); + if (fd == -1 && errno != ENXIO) + throw std::system_error(errno, std::system_category(), "opening lock fifo for writing failed"); + if (fd == -1) { + // No reader was present so we have the exclusive lock! + return true; + } + else { + ::close(fd); + // shared locks exist. Back out. Release exclusive lock on file + unlock(); + return false; + } } - - // We're in shared mode, so opening the fifo means we've successfully acquired - // a shared lock and are done. - ulg.release(); - rw_unlock(); - m_pipe_fd = fd; - return true; -#endif // REALM_FILELOCK_EMULATION -} - -bool File::lock(bool exclusive, bool non_blocking) -{ - REALM_ASSERT_RELEASE(is_attached()); - -#ifdef _WIN32 // Windows version - REALM_ASSERT_RELEASE(!m_have_lock); - - // Under Windows a file lock must be explicitely released before - // the file is closed. It will eventually be released by the - // system, but there is no guarantees on the timing. - - DWORD flags = 0; - if (exclusive) - flags |= LOCKFILE_EXCLUSIVE_LOCK; - if (non_blocking) - flags |= LOCKFILE_FAIL_IMMEDIATELY; - OVERLAPPED overlapped; - memset(&overlapped, 0, sizeof overlapped); - overlapped.Offset = 0; // Just for clarity - overlapped.OffsetHigh = 0; // Just for clarity - if (LockFileEx(m_fd, flags, 0, 1, 0, &overlapped)) { - m_have_lock = true; + else { + // lock shared. We need to release the real exclusive lock on the file, + // but first we must mark the lock as shared by opening the pipe for reading + // Open pipe for reading! + // Due to a bug in iOS 10-12 we need to open in read-write mode or the OS + // will deadlock when un-suspending the app. + int fd = ::open(m_fifo_path.c_str(), O_RDWR | O_NONBLOCK); + if (fd == -1) + throw std::system_error(errno, std::system_category(), "opening lock fifo for reading failed"); + unlock(); + m_pipe_fd = fd; return true; } - DWORD err = GetLastError(); // Eliminate any risk of clobbering - if (err == ERROR_LOCK_VIOLATION) - return false; - throw std::system_error(err, std::system_category(), "LockFileEx() failed"); -#else // _WIN32 + + +#else // BSD / Linux flock() // NOTE: It would probably have been more portable to use fcntl() // based POSIX locks, however these locks are not recursive within // a single process, and since a second attempt to acquire such a @@ -1210,6 +1207,7 @@ bool File::lock(bool exclusive, bool non_blocking) // // Fortunately, on both Linux and Darwin, flock() does not suffer // from this 'spurious unlocking issue'. + int operation = exclusive ? LOCK_EX : LOCK_SH; if (non_blocking) operation |= LOCK_NB; @@ -1221,12 +1219,16 @@ bool File::lock(bool exclusive, bool non_blocking) if (err == EWOULDBLOCK) return false; throw std::system_error(err, std::system_category(), "flock() failed"); + +#endif #endif } + void File::unlock() noexcept { #ifdef _WIN32 // Windows version + if (!m_have_lock) return; @@ -1239,20 +1241,10 @@ void File::unlock() noexcept REALM_ASSERT_RELEASE(r); m_have_lock = false; -#else - // The Linux man page for flock() does not state explicitely that - // unlocking is idempotent, however, we will assume it since there - // is no mention of the error that would be reported if a - // non-locked file were unlocked. - _unlock(m_fd); -#endif -} -void File::rw_unlock() noexcept -{ -#ifndef REALM_FILELOCK_EMULATION - unlock(); #else +#ifdef REALM_FILELOCK_EMULATION + // Coming here with an exclusive lock, we must release that lock. // Coming here with a shared lock, we must close the pipe that we have opened for reading. // - we have to do that under the protection of a proper exclusive lock to serialize @@ -1268,17 +1260,25 @@ void File::rw_unlock() noexcept ::close(m_pipe_fd); m_pipe_fd = -1; } - else { - REALM_ASSERT(m_has_exclusive_lock); - } _unlock(m_fd); m_has_exclusive_lock = false; -#endif // REALM_FILELOCK_EMULATION + +#else // BSD / Linux flock() + + // The Linux man page for flock() does not state explicitely that + // unlocking is idempotent, however, we will assume it since there + // is no mention of the error that would be reported if a + // non-locked file were unlocked. + _unlock(m_fd); + +#endif +#endif } + void* File::map(AccessMode a, size_t size, int /*map_flags*/, size_t offset) const { - return realm::util::mmap({m_fd, m_path, a, m_encryption_key.get()}, size, offset); + return realm::util::mmap(m_fd, size, a, offset, m_encryption_key.get()); } void* File::map_fixed(AccessMode a, void* address, size_t size, int /* map_flags */, size_t offset) const @@ -1307,7 +1307,7 @@ void* File::map_reserve(AccessMode a, size_t size, size_t offset) const #if REALM_ENABLE_ENCRYPTION void* File::map(AccessMode a, size_t size, EncryptedFileMapping*& mapping, int /*map_flags*/, size_t offset) const { - return realm::util::mmap({m_fd, m_path, a, m_encryption_key.get()}, size, offset, mapping); + return realm::util::mmap(m_fd, size, a, offset, m_encryption_key.get(), mapping); } void* File::map_fixed(AccessMode a, void* address, size_t size, EncryptedFileMapping* mapping, int /* map_flags */, @@ -1331,11 +1331,11 @@ void* File::map_reserve(AccessMode a, size_t size, size_t offset, EncryptedFileM { if (m_encryption_key.get()) { // encrypted file - just mmap it, the encryption layer handles if the mapping extends beyond eof - return realm::util::mmap({m_fd, m_path, a, m_encryption_key.get()}, size, offset, mapping); + return realm::util::mmap(m_fd, size, a, offset, m_encryption_key.get(), mapping); } #ifndef _WIN32 // not encrypted, do a proper reservation on Unixes' - return realm::util::mmap_reserve({m_fd, m_path, a, nullptr}, size, offset, mapping); + return realm::util::mmap_reserve(m_fd, size, a, offset, nullptr, mapping); #else // on windows, this is a no-op return nullptr; @@ -1353,7 +1353,7 @@ void File::unmap(void* addr, size_t size) noexcept void* File::remap(void* old_addr, size_t old_size, AccessMode a, size_t new_size, int /*map_flags*/, size_t file_offset) const { - return realm::util::mremap({m_fd, m_path, a, m_encryption_key.get()}, file_offset, old_addr, old_size, new_size); + return realm::util::mremap(m_fd, file_offset, old_addr, old_size, a, new_size, m_encryption_key.get()); } @@ -1365,11 +1365,14 @@ void File::sync_map(FileDesc fd, void* addr, size_t size) bool File::exists(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - return std::filesystem::exists(path); +#ifdef _WIN32 + std::wstring w_path = string_to_wstring(path); + if (_waccess(w_path.c_str(), 0) == 0) + return true; #else // POSIX if (::access(path.c_str(), F_OK) == 0) return true; +#endif int err = errno; // Eliminate any risk of clobbering switch (err) { case EACCES: @@ -1378,15 +1381,12 @@ bool File::exists(const std::string& path) return false; } throw std::system_error(err, std::system_category(), "access() failed"); -#endif } bool File::is_dir(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - return std::filesystem::is_directory(path); -#elif !defined(_WIN32) +#ifndef _WIN32 struct stat statbuf; if (::stat(path.c_str(), &statbuf) == 0) return S_ISDIR(statbuf.st_mode); @@ -1398,6 +1398,9 @@ bool File::is_dir(const std::string& path) return false; } throw std::system_error(err, std::system_category(), "stat() failed"); +#elif REALM_HAVE_STD_FILESYSTEM + std::wstring w_path = string_to_wstring(path); + return std::filesystem::is_directory(w_path); #else static_cast(path); throw util::runtime_error("Not yet supported"); @@ -1417,15 +1420,14 @@ void File::remove(const std::string& path) bool File::try_remove(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - bool result = std::filesystem::remove(path, error); - throwIfFileError(error, path); - return result; +#ifdef _WIN32 + std::wstring w_path = string_to_wstring(path); + if (_wunlink(w_path.c_str()) == 0) + return true; #else // POSIX if (::unlink(path.c_str()) == 0) return true; - +#endif int err = errno; // Eliminate any risk of clobbering std::string msg = get_errno_msg("unlink() failed: ", err); switch (err) { @@ -1440,21 +1442,15 @@ bool File::try_remove(const std::string& path) default: throw AccessError(msg, path); } -#endif } void File::move(const std::string& old_path, const std::string& new_path) { -#if REALM_HAVE_STD_FILESYSTEM - std::error_code error; - std::filesystem::rename(old_path, new_path, error); - - if (error == std::errc::no_such_file_or_directory) { - throw NotFound(error.message(), old_path); - } - throwIfFileError(error, old_path); -#else +#ifdef _WIN32 + // Can't rename to existing file on Windows + try_remove(new_path); +#endif int r = rename(old_path.c_str(), new_path.c_str()); if (r == 0) return; @@ -1474,15 +1470,11 @@ void File::move(const std::string& old_path, const std::string& new_path) default: throw AccessError(msg, old_path); } -#endif } void File::copy(const std::string& origin_path, const std::string& target_path) { -#if REALM_HAVE_STD_FILESYSTEM - std::filesystem::copy_file(origin_path, target_path, std::filesystem::copy_options::overwrite_existing); // Throws -#else File origin_file{origin_path, mode_Read}; // Throws File target_file{target_path, mode_Write}; // Throws size_t buffer_size = 4096; @@ -1493,7 +1485,6 @@ void File::copy(const std::string& origin_path, const std::string& target_path) if (n < buffer_size) break; } -#endif } @@ -1519,7 +1510,29 @@ bool File::compare(const std::string& path_1, const std::string& path_2) bool File::is_same_file_static(FileDesc f1, FileDesc f2) { - return get_unique_id(f1) == get_unique_id(f2); +#if defined(_WIN32) // Windows version + FILE_ID_INFO fi1; + FILE_ID_INFO fi2; + if (GetFileInformationByHandleEx(f1, FILE_INFO_BY_HANDLE_CLASS::FileIdInfo, &fi1, sizeof(fi1))) { + if (GetFileInformationByHandleEx(f2, FILE_INFO_BY_HANDLE_CLASS::FileIdInfo, &fi2, sizeof(fi2))) { + return memcmp(&fi1.FileId, &fi2.FileId, sizeof(fi1.FileId)) == 0 && + fi1.VolumeSerialNumber == fi2.VolumeSerialNumber; + } + } + throw std::system_error(GetLastError(), std::system_category(), "GetFileInformationByHandleEx() failed"); + +#else // POSIX version + + struct stat statbuf; + if (::fstat(f1, &statbuf) == 0) { + dev_t device_id = statbuf.st_dev; + ino_t inode_num = statbuf.st_ino; + if (::fstat(f2, &statbuf) == 0) + return device_id == statbuf.st_dev && inode_num == statbuf.st_ino; + } + throw std::system_error(errno, std::system_category(), "fstat() failed"); + +#endif } bool File::is_same_file(const File& f) const @@ -1532,7 +1545,15 @@ bool File::is_same_file(const File& f) const File::UniqueID File::get_unique_id() const { REALM_ASSERT_RELEASE(is_attached()); - return File::get_unique_id(m_fd); +#ifdef _WIN32 // Windows version + throw util::runtime_error("Not yet supported"); +#else // POSIX version + struct stat statbuf; + if (::fstat(m_fd, &statbuf) == 0) { + return UniqueID(statbuf.st_dev, statbuf.st_ino); + } + throw std::system_error(errno, std::system_category(), "fstat() failed"); +#endif } FileDesc File::get_descriptor() const @@ -1540,67 +1561,52 @@ FileDesc File::get_descriptor() const return m_fd; } -std::optional File::get_unique_id(const std::string& path) +bool File::get_unique_id(const std::string& path, File::UniqueID& uid) { #ifdef _WIN32 // Windows version - // CreateFile2 with creationDisposition OPEN_EXISTING will return a file handle only if the file exists - // otherwise it will raise ERROR_FILE_NOT_FOUND. This call will never create a new file. - WindowsFileHandleHolder fileHandle(::CreateFile2(std::filesystem::path(path).c_str(), FILE_READ_ATTRIBUTES, - FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, - OPEN_EXISTING, nullptr)); - - if (fileHandle == INVALID_HANDLE_VALUE) { - if (GetLastError() == ERROR_FILE_NOT_FOUND) { - return none; - } - throw std::system_error(GetLastError(), std::system_category(), "CreateFileW failed"); - } - - return get_unique_id(fileHandle); + throw std::runtime_error("Not yet supported"); #else // POSIX version struct stat statbuf; if (::stat(path.c_str(), &statbuf) == 0) { - return File::UniqueID(statbuf.st_dev, statbuf.st_ino); + uid.device = statbuf.st_dev; + uid.inode = statbuf.st_ino; + return true; } int err = errno; // Eliminate any risk of clobbering // File doesn't exist if (err == ENOENT) - return none; - throw std::system_error(err, std::system_category(), "stat() failed"); + return false; + throw std::system_error(err, std::system_category(), "fstat() failed"); #endif } -File::UniqueID File::get_unique_id(FileDesc file) +std::string File::get_path() const +{ + return m_path; +} + +bool File::is_removed() const { + REALM_ASSERT_RELEASE(is_attached()); + #ifdef _WIN32 // Windows version - REALM_ASSERT(file != nullptr); - File::UniqueID ret; - if (GetFileInformationByHandleEx(file, FileIdInfo, &ret.id_info, sizeof(ret.id_info)) == 0) { - throw std::system_error(GetLastError(), std::system_category(), "GetFileInformationByHandleEx() failed"); - } - return ret; + return false; // An open file cannot be deleted on Windows + #else // POSIX version - REALM_ASSERT(file >= 0); + struct stat statbuf; - if (::fstat(file, &statbuf) == 0) { - return UniqueID(statbuf.st_dev, statbuf.st_ino); - } + if (::fstat(m_fd, &statbuf) == 0) + return statbuf.st_nlink == 0; throw std::system_error(errno, std::system_category(), "fstat() failed"); + #endif } -std::string File::get_path() const -{ - return m_path; -} std::string File::resolve(const std::string& path, const std::string& base_dir) { -#if REALM_HAVE_STD_FILESYSTEM - std::filesystem::path path_(path.empty() ? "." : path); - return (std::filesystem::path(base_dir) / path_).string(); -#else +#ifndef _WIN32 char dir_sep = '/'; std::string path_2 = path; std::string base_dir_2 = base_dir; @@ -1635,6 +1641,16 @@ std::string File::resolve(const std::string& path, const std::string& base_dir) } */ return base_dir_2 + path_2; +#elif REALM_HAVE_STD_FILESYSTEM + std::filesystem::path path_(path.empty() ? "." : path); + if (path_.is_absolute()) + return path; + + return (std::filesystem::path(base_dir) / path_).string(); +#else + static_cast(path); + static_cast(base_dir); + throw util::runtime_error("Not yet supported"); #endif } @@ -1739,8 +1755,7 @@ bool File::MapBase::try_reserve(const File& file, AccessMode a, size_t size, siz m_offset = offset; #if REALM_ENABLE_ENCRYPTION if (file.m_encryption_key) { - m_encrypted_mapping = - util::reserve_mapping(addr, {m_fd, file.get_path(), a, file.m_encryption_key.get()}, offset); + m_encrypted_mapping = util::reserve_mapping(addr, m_fd, offset, a, file.m_encryption_key.get()); } #endif #endif @@ -1803,17 +1818,23 @@ void File::MapBase::flush() std::time_t File::last_write_time(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - auto time = std::filesystem::last_write_time(path); +#ifdef _WIN32 + auto wpath = string_to_wstring(path); + WindowsFileHandleHolder fileHandle(::CreateFile2(wpath.c_str(), FILE_READ_ATTRIBUTES, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + OPEN_EXISTING, nullptr)); - using namespace std::chrono; -#if __cplusplus >= 202002L - auto system_time = clock_cast(time); -#else - auto system_time = - time_point_cast(time - decltype(time)::clock::now() + system_clock::now()); -#endif - return system_clock::to_time_t(system_time); + if (fileHandle == INVALID_HANDLE_VALUE) { + throw std::system_error(GetLastError(), std::system_category(), "CreateFileW failed"); + } + + FILETIME mtime = {0}; + if (!::GetFileTime(fileHandle, nullptr, nullptr, &mtime)) { + throw std::system_error(GetLastError(), std::system_category(), "GetFileTime failed"); + } + + auto tp = file_time_to_system_clock(mtime); + return std::chrono::system_clock::to_time_t(tp); #else struct stat statbuf; if (::stat(path.c_str(), &statbuf) != 0) { @@ -1825,8 +1846,20 @@ std::time_t File::last_write_time(const std::string& path) File::SizeType File::get_free_space(const std::string& path) { -#if REALM_HAVE_STD_FILESYSTEM - return std::filesystem::space(path).available; +#ifdef _WIN32 + auto pos = path.find_last_of("/\\"); + std::string dir_path; + if (pos != std::string::npos) { + dir_path = path.substr(0, pos); + } + else { + dir_path = path; + } + ULARGE_INTEGER available; + if (!GetDiskFreeSpaceExA(dir_path.c_str(), &available, NULL, NULL)) { + throw std::system_error(errno, std::system_category(), "GetDiskFreeSpaceExA failed"); + } + return available.QuadPart; #else struct statvfs stat; if (statvfs(path.c_str(), &stat) != 0) { @@ -1836,32 +1869,7 @@ File::SizeType File::get_free_space(const std::string& path) #endif } -#if REALM_HAVE_STD_FILESYSTEM - -DirScanner::DirScanner(const std::string& path, bool allow_missing) -{ - try { - m_iterator = std::filesystem::directory_iterator(path); - } - catch (const std::filesystem::filesystem_error& e) { - if (e.code() != std::errc::no_such_file_or_directory || !allow_missing) - throw; - } -} - -DirScanner::~DirScanner() = default; - -bool DirScanner::next(std::string& name) -{ - const std::filesystem::directory_iterator end; - if (m_iterator == end) - return false; - name = m_iterator->path().filename().string(); - m_iterator++; - return true; -} - -#elif !defined(_WIN32) +#ifndef _WIN32 DirScanner::DirScanner(const std::string& path, bool allow_missing) { @@ -1934,6 +1942,32 @@ bool DirScanner::next(std::string& name) } } +#elif REALM_HAVE_STD_FILESYSTEM + +DirScanner::DirScanner(const std::string& path, bool allow_missing) +{ + try { + m_iterator = std::filesystem::directory_iterator(path); + } + catch (const std::filesystem::filesystem_error& e) { + if (e.code() != std::errc::no_such_file_or_directory || !allow_missing) + throw; + } +} + +DirScanner::~DirScanner() = default; + +bool DirScanner::next(std::string& name) +{ + const std::filesystem::directory_iterator end; + if (m_iterator != end) { + name = m_iterator->path().filename().string(); + m_iterator++; + return true; + } + return false; +} + #else DirScanner::DirScanner(const std::string&, bool) diff --git a/src/realm/util/file.hpp b/src/realm/util/file.hpp index 8c3eeaeb722..5b97cb1eb0c 100644 --- a/src/realm/util/file.hpp +++ b/src/realm/util/file.hpp @@ -28,9 +28,7 @@ #include #include -#ifdef _WIN32 -#include -#else +#ifndef _WIN32 #include // POSIX.1-2001 #endif @@ -41,18 +39,24 @@ #include #include -#if defined(_MSVC_LANG) // compiling with MSVC +#if defined(_MSVC_LANG) && _MSVC_LANG >= 201703L // compiling with MSVC and C++ 17 #include #define REALM_HAVE_STD_FILESYSTEM 1 +#if REALM_UWP +// workaround for linker issue described in https://github.com/microsoft/STL/issues/322 +// remove once the Windows SDK or STL fixes this. +#pragma comment(lib, "onecoreuap.lib") +#endif #else #define REALM_HAVE_STD_FILESYSTEM 0 #endif -#if REALM_APPLE_DEVICE && !REALM_TVOS && !REALM_MACCATALYST +#if REALM_APPLE_DEVICE && !REALM_TVOS #define REALM_FILELOCK_EMULATION #endif -namespace realm::util { +namespace realm { +namespace util { class EncryptedFileMapping; @@ -339,34 +343,10 @@ class File { /// Calling this function on an instance that is not attached to /// an open file, or on an instance that is already locked has /// undefined behavior. - void lock(); - - /// Non-blocking version of `lock()`. Returns true if the lock was acquired - /// and false otherwise. - bool try_lock(); - - /// Release a previously acquired lock on this file which was acquired with - /// `lock()` or `try_lock()`. Calling this without holding the lock or - /// while holding a lock acquired with one of the `rw` functions is - /// undefined behavior. - void unlock() noexcept; + void lock_exclusive(); /// Place an shared lock on this file. This blocks the caller - /// until all other locks have been released. - /// - /// Locks acquired on distinct File instances have fully recursive - /// behavior, even if they are acquired in the same process (or - /// thread) and are attached to the same underlying file. - /// - /// Calling this function on an instance that is not attached to an open - /// file, on an instance that is already locked, or on a file which - /// `lock()` (rather than `try_rw_lock_exclusive()` has been called on has - /// undefined behavior. - void rw_lock_shared(); - - /// Attempt to place an exclusive lock on this file. Returns true if the - /// lock could be acquired, and false if an exclusive or shared lock exists - /// for the file. + /// until all other exclusive locks have been released. /// /// Locks acquired on distinct File instances have fully recursive /// behavior, even if they are acquired in the same process (or @@ -375,17 +355,19 @@ class File { /// Calling this function on an instance that is not attached to /// an open file, or on an instance that is already locked has /// undefined behavior. - bool try_rw_lock_exclusive(); + void lock_shared(); + + /// Non-blocking version of lock_exclusive(). Returns true iff it + /// succeeds. + bool try_lock_exclusive(); /// Non-blocking version of lock_shared(). Returns true iff it /// succeeds. - bool try_rw_lock_shared(); + bool try_lock_shared(); - /// Release a previously acquired read-write lock on this file acquired - /// with `rw_lock_shared()`, `try_rw_lock_exclusive()` or - /// `try_rw_lock_shared()`. Calling this after a call to `lock()` or - /// without holding the lock is undefined behavior. - void rw_unlock() noexcept; + /// Release a previously acquired lock on this file. This function + /// is idempotent. + void unlock() noexcept; /// Set the encryption key used for this file. Must be called before any /// mappings are created or any data is read from or written to the file. @@ -535,6 +517,9 @@ class File { bool is_same_file(const File&) const; static bool is_same_file_static(FileDesc f1, FileDesc f2); + // FIXME: Get rid of this method + bool is_removed() const; + /// Resolve the specified path against the specified base directory. /// /// If \a path is absolute, or if \a base_dir is empty, \p path is returned @@ -588,7 +573,7 @@ class File { struct UniqueID { #ifdef _WIN32 // Windows version - FILE_ID_INFO id_info; +// FIXME: This is not implemented for Windows #else UniqueID() : device(0) @@ -614,12 +599,11 @@ class File { // Return the path of the open file, or an empty string if // this file has never been opened. std::string get_path() const; + // Return false if the file doesn't exist. Otherwise uid will be set. + static bool get_unique_id(const std::string& path, UniqueID& uid); - // Return none if the file doesn't exist. Throws on other errors. - static std::optional get_unique_id(const std::string& path); - - // Return the unique id for the file descriptor. Throws if the underlying stat operation fails. - static UniqueID get_unique_id(FileDesc file); + class ExclusiveLock; + class SharedLock; template class Map; @@ -638,7 +622,7 @@ class File { private: #ifdef _WIN32 - HANDLE m_fd = nullptr; + void* m_fd = nullptr; bool m_have_lock = false; // Only valid when m_fd is not null #else int m_fd = -1; @@ -653,7 +637,6 @@ class File { std::string m_path; bool lock(bool exclusive, bool non_blocking); - bool rw_lock(bool exclusive, bool non_blocking); void open_internal(const std::string& path, AccessMode, CreateMode, int flags, bool* success); #ifdef REALM_FILELOCK_EMULATION @@ -671,7 +654,7 @@ class File { FileDesc m_fd; AccessMode m_access_mode = access_ReadOnly; - MapBase() noexcept = default; + MapBase() noexcept; ~MapBase() noexcept; // Disable copying. Copying an opened MapBase will create a scenario @@ -712,6 +695,45 @@ class File { }; +class File::ExclusiveLock { +public: + ExclusiveLock(File& f) + : m_file(f) + { + f.lock_exclusive(); + } + ~ExclusiveLock() noexcept + { + m_file.unlock(); + } + // Disable copying. It is not how this class should be used. + ExclusiveLock(const ExclusiveLock&) = delete; + ExclusiveLock& operator=(const ExclusiveLock&) = delete; + +private: + File& m_file; +}; + +class File::SharedLock { +public: + SharedLock(File& f) + : m_file(f) + { + f.lock_shared(); + } + ~SharedLock() noexcept + { + m_file.unlock(); + } + // Disable copying. It is not how this class should be used. + SharedLock(const SharedLock&) = delete; + SharedLock& operator=(const SharedLock&) = delete; + +private: + File& m_file; +}; + + /// This class provides a RAII abstraction over the concept of a /// memory mapped file. /// @@ -788,10 +810,10 @@ class File::Map : private MapBase { /// See File::remap(). /// - /// Calling this function on a Map instance that is not currently attached - /// to a memory mapped file is equivalent to calling map(). The returned - /// pointer is the same as what will subsequently be returned by - /// get_addr(). + /// Calling this function on a Map instance that is not currently + /// attached to a memory mapped file has undefined behavior. The + /// returned pointer is the same as what will subsequently be + /// returned by get_addr(). T* remap(const File&, AccessMode = access_ReadOnly, size_t size = sizeof(T), int map_flags = 0); /// Try to extend the existing mapping to a given size @@ -881,7 +903,7 @@ class File::UnlockGuard { ~UnlockGuard() noexcept { if (m_file) - m_file->rw_unlock(); + m_file->unlock(); } void release() noexcept { @@ -1132,29 +1154,30 @@ inline bool File::is_attached() const noexcept #endif } -inline void File::rw_lock_shared() +inline void File::lock_exclusive() { - rw_lock(false, false); + lock(true, false); } -inline bool File::try_rw_lock_exclusive() +inline void File::lock_shared() { - return rw_lock(true, true); + lock(false, false); } -inline bool File::try_rw_lock_shared() +inline bool File::try_lock_exclusive() { - return rw_lock(false, true); + return lock(true, true); } -inline void File::lock() +inline bool File::try_lock_shared() { - lock(true, false); + return lock(false, true); } -inline bool File::try_lock() +inline File::MapBase::MapBase() noexcept { - return lock(true, true); + m_addr = nullptr; + m_size = 0; } inline File::MapBase::~MapBase() noexcept @@ -1339,8 +1362,7 @@ inline File::Exists::Exists(const std::string& msg, const std::string& path) inline bool operator==(const File::UniqueID& lhs, const File::UniqueID& rhs) { #ifdef _WIN32 // Windows version - return lhs.id_info.VolumeSerialNumber == rhs.id_info.VolumeSerialNumber && - memcmp(&lhs.id_info.FileId, &rhs.id_info.FileId, sizeof(lhs.id_info.FileId)) == 0; + throw util::runtime_error("Not yet supported"); #else // POSIX version return lhs.device == rhs.device && lhs.inode == rhs.inode; #endif @@ -1354,9 +1376,7 @@ inline bool operator!=(const File::UniqueID& lhs, const File::UniqueID& rhs) inline bool operator<(const File::UniqueID& lhs, const File::UniqueID& rhs) { #ifdef _WIN32 // Windows version - if (lhs.id_info.VolumeSerialNumber != rhs.id_info.VolumeSerialNumber) - return lhs.id_info.VolumeSerialNumber < rhs.id_info.VolumeSerialNumber; - return memcmp(&lhs.id_info.FileId, &rhs.id_info.FileId, sizeof(lhs.id_info.FileId)) < 0; + throw util::runtime_error("Not yet supported"); #else // POSIX version if (lhs.device < rhs.device) return true; @@ -1383,6 +1403,7 @@ inline bool operator>=(const File::UniqueID& lhs, const File::UniqueID& rhs) return !(lhs < rhs); } -} // namespace realm::util +} // namespace util +} // namespace realm #endif // REALM_UTIL_FILE_HPP diff --git a/src/realm/util/file_mapper.cpp b/src/realm/util/file_mapper.cpp index 15eeae05832..d09f3a0111a 100644 --- a/src/realm/util/file_mapper.cpp +++ b/src/realm/util/file_mapper.cpp @@ -103,7 +103,7 @@ struct mappings_for_file { // Group the information we need to map a SIGSEGV address to an // EncryptedFileMapping for the sake of cache-friendliness with 3+ active -// mappings (and no worse with only two) +// mappings (and no worse with only two struct mapping_and_addr { std::shared_ptr mapping; void* addr; @@ -499,18 +499,19 @@ SharedFileInfo* get_file_info_for_file(File& file) namespace { -EncryptedFileMapping* add_mapping(void* addr, size_t size, const FileAttributes& file, size_t file_offset) +EncryptedFileMapping* add_mapping(void* addr, size_t size, FileDesc fd, size_t file_offset, File::AccessMode access, + const char* encryption_key) { #ifndef _WIN32 struct stat st; - if (fstat(file.fd, &st)) { + if (fstat(fd, &st)) { int err = errno; // Eliminate any risk of clobbering throw std::system_error(err, std::system_category(), "fstat() failed"); } #endif - size_t fs = to_size_t(File::get_size_static(file.fd)); + size_t fs = to_size_t(File::get_size_static(fd)); if (fs > 0 && fs < page_size()) throw DecryptionFailed(); @@ -519,7 +520,7 @@ EncryptedFileMapping* add_mapping(void* addr, size_t size, const FileAttributes& std::vector::iterator it; for (it = mappings_by_file.begin(); it != mappings_by_file.end(); ++it) { #ifdef _WIN32 - if (File::is_same_file_static(it->handle, file.fd)) + if (File::is_same_file_static(it->handle, fd)) break; #else if (it->inode == st.st_ino && it->device == st.st_dev) @@ -533,22 +534,22 @@ EncryptedFileMapping* add_mapping(void* addr, size_t size, const FileAttributes& if (it == mappings_by_file.end()) { mappings_by_file.reserve(mappings_by_file.size() + 1); mappings_for_file f; - f.info = std::make_shared(reinterpret_cast(file.encryption_key)); + f.info = std::make_shared(reinterpret_cast(encryption_key)); - FileDesc fd_duped; #ifdef _WIN32 - if (!DuplicateHandle(GetCurrentProcess(), file.fd, GetCurrentProcess(), &fd_duped, 0, FALSE, - DUPLICATE_SAME_ACCESS)) + FileDesc fd2; + if (!DuplicateHandle(GetCurrentProcess(), fd, GetCurrentProcess(), &fd2, 0, FALSE, DUPLICATE_SAME_ACCESS)) throw std::system_error(GetLastError(), std::system_category(), "DuplicateHandle() failed"); - f.info->fd = f.handle = fd_duped; + fd = fd2; + f.info->fd = f.handle = fd; #else - fd_duped = dup(file.fd); + fd = dup(fd); - if (fd_duped == -1) { + if (fd == -1) { int err = errno; // Eliminate any risk of clobbering throw std::system_error(err, std::system_category(), "dup() failed"); } - f.info->fd = fd_duped; + f.info->fd = fd; f.device = st.st_dev; f.inode = st.st_ino; #endif @@ -557,14 +558,14 @@ EncryptedFileMapping* add_mapping(void* addr, size_t size, const FileAttributes& it = mappings_by_file.end() - 1; } else { - it->info->cryptor.check_key(reinterpret_cast(file.encryption_key)); + it->info->cryptor.check_key(reinterpret_cast(encryption_key)); } try { mapping_and_addr m; m.addr = addr; m.size = size; - m.mapping = std::make_shared(*it->info, file_offset, addr, size, file.access); + m.mapping = std::make_shared(*it->info, file_offset, addr, size, access); mappings_by_addr.push_back(m); // can't throw due to reserve() above return m.mapping.get(); } @@ -611,25 +612,27 @@ void remove_mapping(void* addr, size_t size) } } // anonymous namespace -void* mmap(const FileAttributes& file, size_t size, size_t offset, EncryptedFileMapping*& mapping) +void* mmap(FileDesc fd, size_t size, File::AccessMode access, size_t offset, const char* encryption_key, + EncryptedFileMapping*& mapping) { _impl::SimulatedFailure::trigger_mmap(size); - if (file.encryption_key) { + if (encryption_key) { size = round_up_to_page_size(size); void* addr = mmap_anon(size); - mapping = add_mapping(addr, size, file, offset); + mapping = add_mapping(addr, size, fd, offset, access, encryption_key); return addr; } else { mapping = nullptr; - return mmap(file, size, offset); + return mmap(fd, size, access, offset, nullptr); } } -EncryptedFileMapping* reserve_mapping(void* addr, const FileAttributes& file, size_t offset) +EncryptedFileMapping* reserve_mapping(void* addr, FileDesc fd, size_t offset, File::AccessMode access, + const char* encryption_key) { - return add_mapping(addr, 0, file, offset); + return add_mapping(addr, 0, fd, offset, access, encryption_key); } void extend_encrypted_mapping(EncryptedFileMapping* mapping, void* addr, size_t offset, size_t old_size, @@ -647,15 +650,15 @@ void remove_encrypted_mapping(void* addr, size_t size) remove_mapping(addr, size); } -void* mmap_reserve(const FileAttributes& file, size_t reservation_size, size_t offset_in_file, - EncryptedFileMapping*& mapping) +void* mmap_reserve(FileDesc fd, size_t reservation_size, File::AccessMode access, size_t offset_in_file, + const char* enc_key, EncryptedFileMapping*& mapping) { - auto addr = mmap_reserve(file.fd, reservation_size, offset_in_file); - if (file.encryption_key) { + auto addr = mmap_reserve(fd, reservation_size, offset_in_file); + if (enc_key) { REALM_ASSERT(reservation_size == round_up_to_page_size(reservation_size)); // we create a mapping for the entire reserved area. This causes full initialization of some fairly // large std::vectors, which it would be nice to avoid. This is left as a future optimization. - mapping = add_mapping(addr, reservation_size, file, offset_in_file); + mapping = add_mapping(addr, reservation_size, fd, offset_in_file, access, enc_key); } else { mapping = nullptr; @@ -781,25 +784,25 @@ void* mmap_reserve(FileDesc fd, size_t reservation_size, size_t offset_in_file) } -void* mmap(const FileAttributes& file, size_t size, size_t offset) +void* mmap(FileDesc fd, size_t size, File::AccessMode access, size_t offset, const char* encryption_key) { _impl::SimulatedFailure::trigger_mmap(size); #if REALM_ENABLE_ENCRYPTION - if (file.encryption_key) { + if (encryption_key) { size = round_up_to_page_size(size); void* addr = mmap_anon(size); - add_mapping(addr, size, file, offset); + add_mapping(addr, size, fd, offset, access, encryption_key); return addr; } else #else - REALM_ASSERT(!file.encryption_key); + REALM_ASSERT(!encryption_key); #endif { #ifndef _WIN32 int prot = PROT_READ; - switch (file.access) { + switch (access) { case File::access_ReadWrite: prot |= PROT_WRITE; break; @@ -807,7 +810,7 @@ void* mmap(const FileAttributes& file, size_t size, size_t offset) break; } - void* addr = ::mmap(nullptr, size, prot, MAP_SHARED, file.fd, offset); + void* addr = ::mmap(nullptr, size, prot, MAP_SHARED, fd, offset); if (addr != MAP_FAILED) return addr; @@ -826,7 +829,7 @@ void* mmap(const FileAttributes& file, size_t size, size_t offset) DWORD protect = PAGE_READONLY; DWORD desired_access = FILE_MAP_READ; - switch (file.access) { + switch (access) { case File::access_ReadOnly: break; case File::access_ReadWrite: @@ -837,7 +840,7 @@ void* mmap(const FileAttributes& file, size_t size, size_t offset) LARGE_INTEGER large_int; if (int_cast_with_overflow_detect(offset + size, large_int.QuadPart)) throw std::runtime_error("Map size is too large"); - HANDLE map_handle = CreateFileMappingFromApp(file.fd, 0, protect, offset + size, nullptr); + HANDLE map_handle = CreateFileMappingFromApp(fd, 0, protect, offset + size, nullptr); if (!map_handle) throw AddressSpaceExhausted(get_errno_msg("CreateFileMapping() failed: ", GetLastError()) + " size: " + util::to_string(size) + " offset: " + util::to_string(offset)); @@ -876,10 +879,11 @@ void munmap(void* addr, size_t size) #endif } -void* mremap(const FileAttributes& file, size_t file_offset, void* old_addr, size_t old_size, size_t new_size) +void* mremap(FileDesc fd, size_t file_offset, void* old_addr, size_t old_size, File::AccessMode a, size_t new_size, + const char* encryption_key) { #if REALM_ENABLE_ENCRYPTION - if (file.encryption_key) { + if (encryption_key) { LockGuard lock(mapping_mutex); size_t rounded_old_size = round_up_to_page_size(old_size); if (mapping_and_addr* m = find_mapping_for_addr(old_addr, rounded_old_size)) { @@ -908,6 +912,8 @@ void* mremap(const FileAttributes& file, size_t file_offset, void* old_addr, siz // the encryption key which is an error. REALM_UNREACHABLE(); } +#else + static_cast(encryption_key); #endif #ifdef _GNU_SOURCE @@ -931,7 +937,7 @@ void* mremap(const FileAttributes& file, size_t file_offset, void* old_addr, siz } #endif - void* new_addr = mmap(file, new_size, file_offset); + void* new_addr = mmap(fd, new_size, a, file_offset, nullptr); #ifdef _WIN32 if (!UnmapViewOfFile(old_addr)) diff --git a/src/realm/util/file_mapper.hpp b/src/realm/util/file_mapper.hpp index 2756e80c4cd..2a9441955b9 100644 --- a/src/realm/util/file_mapper.hpp +++ b/src/realm/util/file_mapper.hpp @@ -28,19 +28,13 @@ namespace realm { namespace util { -struct FileAttributes { - FileDesc fd; - std::string path; - File::AccessMode access; - const char* encryption_key = nullptr; -}; - -void* mmap(const FileAttributes& file, size_t size, size_t offset); +void* mmap(FileDesc fd, size_t size, File::AccessMode access, size_t offset, const char* encryption_key); void* mmap_fixed(FileDesc fd, void* address_request, size_t size, File::AccessMode access, size_t offset, const char* enc_key); void* mmap_reserve(FileDesc fd, size_t size, size_t offset); void munmap(void* addr, size_t size); -void* mremap(const FileAttributes& file, size_t file_offset, void* old_addr, size_t old_size, size_t new_size); +void* mremap(FileDesc fd, size_t file_offset, void* old_addr, size_t old_size, File::AccessMode a, size_t new_size, + const char* encryption_key); void msync(FileDesc fd, void* addr, size_t size); void* mmap_anon(size_t size); @@ -98,13 +92,16 @@ SharedFileInfo* get_file_info_for_file(File& file); // This variant allows the caller to obtain direct access to the encrypted file mapping // for optimization purposes. -void* mmap(const FileAttributes& file, size_t size, size_t offset, EncryptedFileMapping*& mapping); +void* mmap(FileDesc fd, size_t size, File::AccessMode access, size_t offset, const char* encryption_key, + EncryptedFileMapping*& mapping); void* mmap_fixed(FileDesc fd, void* address_request, size_t size, File::AccessMode access, size_t offset, const char* enc_key, EncryptedFileMapping* mapping); -void* mmap_reserve(const FileAttributes& file, size_t size, size_t offset, EncryptedFileMapping*& mapping); +void* mmap_reserve(FileDesc fd, size_t size, File::AccessMode am, size_t offset, const char* enc_key, + EncryptedFileMapping*& mapping); -EncryptedFileMapping* reserve_mapping(void* addr, const FileAttributes& file, size_t offset); +EncryptedFileMapping* reserve_mapping(void* addr, FileDesc fd, size_t offset, File::AccessMode access, + const char* encryption_key); void extend_encrypted_mapping(EncryptedFileMapping* mapping, void* addr, size_t offset, size_t old_size, size_t new_size); diff --git a/src/realm/util/interprocess_mutex.hpp b/src/realm/util/interprocess_mutex.hpp index f5626b27650..1e2f7130f13 100644 --- a/src/realm/util/interprocess_mutex.hpp +++ b/src/realm/util/interprocess_mutex.hpp @@ -242,8 +242,7 @@ inline void InterprocessMutex::set_shared_part(SharedPart& shared_part, const st std::lock_guard guard(*s_mutex); // Try to get the file uid if the file exists - if (auto uid = File::get_unique_id(m_filename)) { - m_fileuid = std::move(*uid); + if (File::get_unique_id(m_filename, m_fileuid)) { auto result = s_info_map->find(m_fileuid); if (result != s_info_map->end()) { // File exists and the lock info has been created in the map. @@ -327,7 +326,7 @@ inline void InterprocessMutex::lock() { #if REALM_ROBUST_MUTEX_EMULATION std::unique_lock mutex_lock(m_lock_info->m_local_mutex); - m_lock_info->m_file.lock(); + m_lock_info->m_file.lock_exclusive(); mutex_lock.release(); #else @@ -348,7 +347,7 @@ inline bool InterprocessMutex::try_lock() if (!mutex_lock.owns_lock()) { return false; } - bool success = m_lock_info->m_file.try_lock(); + bool success = m_lock_info->m_file.try_lock_exclusive(); if (success) { mutex_lock.release(); return true; diff --git a/src/realm/util/logger.cpp b/src/realm/util/logger.cpp index 0949e98efa3..58cfb3c016c 100644 --- a/src/realm/util/logger.cpp +++ b/src/realm/util/logger.cpp @@ -22,8 +22,6 @@ namespace realm::util { -const Logger::Level Logger::default_log_level = Level::info; - const char* Logger::get_level_prefix(Level level) noexcept { switch (level) { @@ -66,13 +64,21 @@ void ThreadSafeLogger::do_log(Level level, const std::string& message) Logger::do_log(*m_base_logger_ptr, level, message); // Throws } -void PrefixLogger::do_log(Level level, const std::string& message) +Logger::Level ThreadSafeLogger::get_level_threshold() noexcept { - Logger::do_log(m_chained_logger, level, m_prefix + message); // Throws + LockGuard l(m_mutex); + return Logger::get_level_threshold(); } -void LocalThresholdLogger::do_log(Logger::Level level, std::string const& message) +void ThreadSafeLogger::set_level_threshold(Level level) noexcept { - Logger::do_log(*m_chained_logger, level, message); // Throws + LockGuard l(m_mutex); + Logger::set_level_threshold(level); } + +void PrefixLogger::do_log(Level level, const std::string& message) +{ + Logger::do_log(m_base_logger, level, m_prefix + message); // Throws +} + } // namespace realm::util diff --git a/src/realm/util/logger.hpp b/src/realm/util/logger.hpp index dcb4621ca95..b1c13608946 100644 --- a/src/realm/util/logger.hpp +++ b/src/realm/util/logger.hpp @@ -33,13 +33,12 @@ namespace realm::util { /// All messages logged with a level that is lower than the current threshold /// will be dropped. For the sake of efficiency, this test happens before the /// message is formatted. This class allows for the log level threshold to be -/// changed over time and any subclasses will share the same reference to a -/// log level threshold instance. The default log level threshold is -/// Logger::Level::info and is defined by Logger::default_log_level. +/// changed over time. The initial log level threshold is Logger::Level::info. /// -/// The Logger threshold level is intrinsically thread safe since it uses an -/// atomic to store the value. However, the do_log() operation is not, so it -/// is up to the subclass to ensure thread safety of the output operation. +/// A logger is not inherently thread-safe, but specific implementations can be +/// (see ThreadSafeLogger). For a logger to be thread-safe, the implementation +/// of do_log() must be thread-safe and the referenced LevelThreshold object +/// must have a thread-safe get() method. /// /// Examples: /// @@ -75,51 +74,35 @@ class Logger { /// attention to efficiency. /// trace A version of 'debug' that allows for very high volume /// output. - // equivalent to realm_log_level_e in realm.h and must be kept in sync - - // this is enforced in logging.cpp. + // equivalent to realm_log_level_e in realm.h and must be kept in sync enum class Level { all = 0, trace = 1, debug = 2, detail = 3, info = 4, warn = 5, error = 6, fatal = 7, off = 8 }; - static const Level default_log_level; - template void log(Level, const char* message, Params&&...); - virtual Level get_level_threshold() const noexcept + virtual Level get_level_threshold() noexcept { - // Don't need strict ordering, mainly that the gets/sets are atomic - return m_level_threshold.load(std::memory_order_relaxed); + return m_level_threshold; } virtual void set_level_threshold(Level level) noexcept { - // Don't need strict ordering, mainly that the gets/sets are atomic - m_level_threshold.store(level, std::memory_order_relaxed); + m_level_threshold = level; } /// Shorthand for `int(level) >= int(m_level_threshold)`. inline bool would_log(Level level) const noexcept { - return static_cast(level) >= static_cast(get_level_threshold()); + return static_cast(level) >= static_cast(m_level_threshold); } virtual inline ~Logger() noexcept = default; protected: - Logger() noexcept - : m_threshold_base{Logger::default_log_level} - , m_level_threshold{m_threshold_base} - { - } + Logger() noexcept = default; - explicit Logger(Level level) noexcept - : m_threshold_base{level} - , m_level_threshold{m_threshold_base} - { - } - - explicit Logger(const std::shared_ptr& base_logger) noexcept - : m_base_logger_ptr{base_logger} - , m_level_threshold{m_base_logger_ptr->m_level_threshold} + Logger(Level level) noexcept + : m_level_threshold{level} { } @@ -129,17 +112,7 @@ class Logger { static const char* get_level_prefix(Level) noexcept; -private: - // Only used by the base Logger class - std::atomic m_threshold_base; - -protected: - // Used by subclasses that link to a base logger - std::shared_ptr m_base_logger_ptr; - - // Shared level threshold for subclasses that link to a base logger - // See PrefixLogger and ThreadSafeLogger - std::atomic& m_level_threshold; + Level m_level_threshold = Logger::Level::info; private: template @@ -152,8 +125,12 @@ std::basic_ostream& operator<<(std::basic_ostream&, Logger::Level); template std::basic_istream& operator>>(std::basic_istream&, Logger::Level&); -/// A logger that writes to STDERR, which is thread safe. -/// Since this class is a subclass of Logger, it maintains its own modifiable log + +/// A logger that writes to STDERR, which is thread safe. However, the setting +/// the threshold level is not thread safe. Wrap this class in a ThreadSafeLogger +/// to make this class fully thread safe. +/// +/// Since this class is a subclass of Logger, it contains a modifiable log /// level threshold. class StderrLogger : public Logger { public: @@ -171,7 +148,7 @@ class StderrLogger : public Logger { /// A logger that writes to a stream. This logger is not thread-safe. /// -/// Since this class is a subclass of Logger, it maintains its own modifiable log +/// Since this class is a subclass of Logger, it contains a modifiable log /// level threshold. class StreamLogger : public Logger { public: @@ -185,9 +162,9 @@ class StreamLogger : public Logger { }; -/// A logger that writes to a new file. This logger is not thread-safe. +/// A logger that writes to a file. This logger is not thread-safe. /// -/// Since this class is a subclass of Logger, it maintains its own thread safe log +/// Since this class is a subclass of Logger, it contains a modifiable log /// level threshold. class FileLogger : public StreamLogger { public: @@ -200,10 +177,6 @@ class FileLogger : public StreamLogger { std::ostream m_out; }; -/// A logger that appends to a file. This logger is not thread-safe. -/// -/// Since this class is a subclass of Logger, it maintains its own thread safe log -/// level threshold. class AppendToFileLogger : public StreamLogger { public: explicit AppendToFileLogger(std::string path); @@ -216,64 +189,43 @@ class AppendToFileLogger : public StreamLogger { }; -/// A thread-safe logger where do_log() is thread safe. The log level is already -/// thread safe since Logger uses an atomic to store the log level threshold. +/// A thread-safe logger. This logger ignores the level threshold of the base +/// logger. Instead, it introduces new a LevelThreshold object with a fixed +/// value to achieve thread safety. class ThreadSafeLogger : public Logger { public: explicit ThreadSafeLogger(const std::shared_ptr& base_logger) noexcept; + Level get_level_threshold() noexcept override; + void set_level_threshold(Level level) noexcept override; + protected: void do_log(Level, const std::string&) final; private: + std::shared_ptr m_base_logger_ptr; // bind for the lifetime of this logger Mutex m_mutex; }; -/// A logger that adds a fixed prefix to each message. +/// A logger that adds a fixed prefix to each message. This logger is +/// thread-safe if, and only if the base logger is thread-safe. class PrefixLogger : public Logger { public: - // A PrefixLogger must initially be created from a base Logger shared_ptr PrefixLogger(std::string prefix, const std::shared_ptr& base_logger) noexcept; - - // Used for chaining a series of prefixes together for logging that combines prefix values - PrefixLogger(std::string prefix, PrefixLogger& chained_logger) noexcept; + PrefixLogger(std::string prefix, Logger& base_logger) noexcept; protected: void do_log(Level, const std::string&) final; private: const std::string m_prefix; - // The next logger in the chain for chained PrefixLoggers or the base_logger - Logger& m_chained_logger; + std::shared_ptr m_base_logger_ptr; // bind for the lifetime of this logger + Logger& m_base_logger; }; -// Logger with a local log level that is independent of the parent log level threshold -// The LocalThresholdLogger will define its own atomic log level threshold and -// will be unaffected by changes to the log level threshold of the parent. -// In addition, any changes to the log level threshold of this class or any -// subsequent linked loggers will not change the log level threshold of the -// parent. The parent will only be used for outputting log messages. -class LocalThresholdLogger : public Logger { -public: - // A shared_ptr parent must be provided for this class for log output - // Local log level is initialized with the current value from the provided logger - LocalThresholdLogger(const std::shared_ptr&); - - // A shared_ptr parent must be provided for this class for log output - LocalThresholdLogger(const std::shared_ptr&, Level); - - void do_log(Logger::Level level, std::string const& message) override; - -protected: - std::shared_ptr m_chained_logger; -}; - - -/// A logger that essentially performs a noop when logging functions are called -/// The log level threshold for this logger is always Logger::Level::off and -/// cannot be changed. +// A logger that essentially performs a noop when logging functions are called class NullLogger : public Logger { public: NullLogger() @@ -281,7 +233,7 @@ class NullLogger : public Logger { { } - Level get_level_threshold() const noexcept override + Level get_level_threshold() noexcept override { return Level::off; } @@ -486,44 +438,24 @@ inline AppendToFileLogger::AppendToFileLogger(util::File file) } inline ThreadSafeLogger::ThreadSafeLogger(const std::shared_ptr& base_logger) noexcept - : Logger(base_logger) + : Logger(base_logger->get_level_threshold()) + , m_base_logger_ptr(base_logger) { } -// Construct a PrefixLogger from another PrefixLogger object for chaining the prefixes on log output -inline PrefixLogger::PrefixLogger(std::string prefix, PrefixLogger& prefix_logger) noexcept - // Save an alias of the base_logger shared_ptr from the passed in PrefixLogger - : Logger(prefix_logger.m_base_logger_ptr) +inline PrefixLogger::PrefixLogger(std::string prefix, Logger& base_logger) noexcept + : Logger(base_logger.get_level_threshold()) , m_prefix{std::move(prefix)} - , m_chained_logger{prefix_logger} // do_log() writes to the chained logger + , m_base_logger{base_logger} { } -// Construct a PrefixLogger from any Logger shared_ptr (PrefixLogger, StdErrLogger, etc.) -// The first PrefixLogger must always be created from a Logger shared_ptr, subsequent PrefixLoggers -// created, will point back to this logger shared_ptr for referencing the level_threshold when -// logging output. inline PrefixLogger::PrefixLogger(std::string prefix, const std::shared_ptr& base_logger) noexcept - : Logger(base_logger) // Save an alias of the passed in base_logger shared_ptr - , m_prefix{std::move(prefix)} - , m_chained_logger{*base_logger} // do_log() writes to the chained logger -{ -} - -// Construct a LocalThresholdLogger using the current log level value from the parent -inline LocalThresholdLogger::LocalThresholdLogger(const std::shared_ptr& base_logger) : Logger(base_logger->get_level_threshold()) - , m_chained_logger{base_logger} -{ -} - -// Construct a LocalThresholdLogger using the provided log level threshold value -inline LocalThresholdLogger::LocalThresholdLogger(const std::shared_ptr& base_logger, Level threshold) - : Logger(threshold) - , m_chained_logger{base_logger} + , m_prefix{std::move(prefix)} + , m_base_logger_ptr(base_logger) + , m_base_logger{*m_base_logger_ptr} { - // Verify the passed in shared ptr is not null - REALM_ASSERT(m_chained_logger); } } // namespace realm::util diff --git a/src/realm/util/misc_errors.cpp b/src/realm/util/misc_errors.cpp index b576f3414d1..4d94442df3d 100644 --- a/src/realm/util/misc_errors.cpp +++ b/src/realm/util/misc_errors.cpp @@ -30,6 +30,8 @@ class misc_category : public std::error_category { std::string message(int) const override; }; +misc_category g_misc_category; + const char* misc_category::name() const noexcept { return "tigthdb.misc"; @@ -54,13 +56,7 @@ namespace error { std::error_code make_error_code(misc_errors err) { - return std::error_code(err, misc_error_category()); -} - -const std::error_category& misc_error_category() -{ - static misc_category misc_category; - return misc_category; + return std::error_code(err, g_misc_category); } } // namespace error diff --git a/src/realm/util/misc_errors.hpp b/src/realm/util/misc_errors.hpp index de71608bea3..9335ba90bd5 100644 --- a/src/realm/util/misc_errors.hpp +++ b/src/realm/util/misc_errors.hpp @@ -31,7 +31,6 @@ enum misc_errors { }; std::error_code make_error_code(misc_errors); -const std::error_category& misc_error_category(); } // namespace error } // namespace util diff --git a/src/realm/util/timestamp_logger.hpp b/src/realm/util/timestamp_logger.hpp index cbcd72d6d55..b3d7173d276 100644 --- a/src/realm/util/timestamp_logger.hpp +++ b/src/realm/util/timestamp_logger.hpp @@ -14,7 +14,7 @@ class TimestampStderrLogger : public Logger { using Precision = TimestampFormatter::Precision; using Config = TimestampFormatter::Config; - explicit TimestampStderrLogger(Config = {}, Level = Logger::default_log_level); + explicit TimestampStderrLogger(Config = {}, Level = Level::info); protected: void do_log(Logger::Level, const std::string& message) final; diff --git a/src/realm/utilities.hpp b/src/realm/utilities.hpp index 239dff61262..31d94e1d50e 100644 --- a/src/realm/utilities.hpp +++ b/src/realm/utilities.hpp @@ -387,7 +387,7 @@ struct PlacementDelete { }; #ifdef _WIN32 -typedef HANDLE FileDesc; +typedef void* FileDesc; #else typedef int FileDesc; #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f5ee6bcc9a8..987f701a37b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2,10 +2,9 @@ add_subdirectory(util) add_custom_target(benchmarks) add_subdirectory(object-store) -# AFL and LIBFUZZER not yet supported by Windows +# AFL not yet supported by Windows if(NOT CMAKE_SYSTEM_NAME MATCHES "^Windows") add_subdirectory(fuzzy) - add_subdirectory(realm-fuzzer) endif() add_subdirectory(benchmark-common-tasks) diff --git a/test/benchmark-sync/bench_transform.cpp b/test/benchmark-sync/bench_transform.cpp index 9e3ff7b649b..a94ed74afb2 100644 --- a/test/benchmark-sync/bench_transform.cpp +++ b/test/benchmark-sync/bench_transform.cpp @@ -95,7 +95,6 @@ void transform_transactions(TestContext& test_context) fixture.start_client(1); session_2.wait_for_upload_complete_or_client_stopped(); session_2.wait_for_download_complete_or_client_stopped(); - session_2.detach(); fixture.stop_client(1); // Upload changes of first client and wait to integrate changes from second client. @@ -176,7 +175,6 @@ void transform_instructions(TestContext& test_context) fixture.start_client(1); session_2.wait_for_upload_complete_or_client_stopped(); session_2.wait_for_download_complete_or_client_stopped(); - session_2.detach(); fixture.stop_client(1); // Upload changes of first client and wait to integrate changes from second client. @@ -255,7 +253,6 @@ void connected_objects(TestContext& test_context) fixture.start_client(1); session_2.wait_for_upload_complete_or_client_stopped(); session_2.wait_for_download_complete_or_client_stopped(); - session_2.detach(); fixture.stop_client(1); // Upload changes of first client and wait to integrate changes from second client. diff --git a/test/fuzzy/libfuzzer_entry.cpp b/test/fuzzy/libfuzzer_entry.cpp index 67920aca125..db267ef0913 100644 --- a/test/fuzzy/libfuzzer_entry.cpp +++ b/test/fuzzy/libfuzzer_entry.cpp @@ -1,15 +1,19 @@ #include #include + +#include +#include +#include +#include + #include "../fuzz_group.hpp" #include "../util/test_path.hpp" using namespace realm; using namespace realm::util; -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size); - // This function is the entry point for libfuzzer, main is auto-generated -int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) { if (Size == 0) { return 0; @@ -17,7 +21,8 @@ int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) realm::test_util::RealmPathInfo test_context{"libfuzzer_test"}; SHARED_GROUP_TEST_PATH(path); disable_sync_to_disk(); + util::Optional log; // logging off std::string contents(reinterpret_cast(Data), Size); - parse_and_apply_instructions(contents, path, nullptr); + parse_and_apply_instructions(contents, path, log); return 0; // Non-zero return values are reserved for future use. } diff --git a/test/object-store/audit.cpp b/test/object-store/audit.cpp index 8cfb2b746eb..3f4c75cc96e 100644 --- a/test/object-store/audit.cpp +++ b/test/object-store/audit.cpp @@ -420,9 +420,9 @@ TEST_CASE("audit object serialization") { populate_object(obj); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -446,22 +446,22 @@ TEST_CASE("audit object serialization") { serializer->expected_obj = &obj1; - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object(realm, obj1); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); REQUIRE(serializer->completion_count == 1); - scope = audit->begin_scope("empty scope"); - audit->end_scope(scope, assert_no_error); + audit->begin_scope("empty scope"); + audit->end_scope(assert_no_error); audit->wait_for_completion(); REQUIRE(serializer->completion_count == 2); serializer->expected_obj = &obj2; - scope = audit->begin_scope("scope 2"); + audit->begin_scope("scope 2"); Object(realm, obj2); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); REQUIRE(serializer->completion_count == 3); @@ -484,9 +484,9 @@ TEST_CASE("audit object serialization") { realm->begin_transaction(); auto obj = table->create_object_with_primary_key(2); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object(realm, obj); - audit->end_scope(scope, [](auto error) { + audit->end_scope([](auto error) { REQUIRE(error); REQUIRE_THROWS_CONTAINING(std::rethrow_exception(error), "custom serialization error"); }); @@ -495,12 +495,12 @@ TEST_CASE("audit object serialization") { SECTION("write transaction serialization") { SECTION("create object") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); auto obj = table->create_object_with_primary_key(2); populate_object(obj); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -529,12 +529,12 @@ TEST_CASE("audit object serialization") { populate_object(obj); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); obj.set("int", 3); obj.set("bool", true); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -557,11 +557,11 @@ TEST_CASE("audit object serialization") { populate_object(obj); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); obj.remove(); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -579,11 +579,11 @@ TEST_CASE("audit object serialization") { obj.create_and_set_linked_object(obj.get_table()->get_column_key("embedded object")).set_all(100); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); obj.get_linked_object("embedded object").remove(); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -603,7 +603,7 @@ TEST_CASE("audit object serialization") { objects.push_back(target_table->create_object_with_primary_key(i).set_all(i)); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); // Mutate then delete should not report the mutate @@ -621,7 +621,7 @@ TEST_CASE("audit object serialization") { obj2.remove(); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -638,10 +638,10 @@ TEST_CASE("audit object serialization") { } SECTION("empty write transactions do not produce an event") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); REQUIRE(get_audit_events(test_session).empty()); @@ -649,9 +649,9 @@ TEST_CASE("audit object serialization") { } SECTION("empty query") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where()).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); REQUIRE(get_audit_events(test_session).empty()); } @@ -665,9 +665,9 @@ TEST_CASE("audit object serialization") { realm->commit_transaction(); SECTION("query counts as a read on all objects matching the query") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where().less(table->get_column_key("_id"), 5)).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); REQUIRE(events.size() == 1); @@ -675,11 +675,11 @@ TEST_CASE("audit object serialization") { } SECTION("subsequent reads on the same table are folded into the query") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where().less(table->get_column_key("_id"), 5)).snapshot(); Object(realm, table->get_object(3)); // does not produce any new audit data Object(realm, table->get_object(7)); // adds this object to the query's event - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); REQUIRE(events.size() == 1); @@ -687,10 +687,10 @@ TEST_CASE("audit object serialization") { } SECTION("reads on different tables are not folded into query") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where().less(table->get_column_key("_id"), 5)).snapshot(); Object(realm, target_table->get_object(3)); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); REQUIRE(events.size() == 2); @@ -699,11 +699,11 @@ TEST_CASE("audit object serialization") { } SECTION("reads on same table following a read on a different table are not folded into query") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where().less(table->get_column_key("_id"), 5)).snapshot(); Object(realm, target_table->get_object(3)); Object(realm, table->get_object(3)); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); REQUIRE(events.size() == 3); @@ -713,12 +713,12 @@ TEST_CASE("audit object serialization") { } SECTION("reads with intervening writes are not combined") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where().less(table->get_column_key("_id"), 5)).snapshot(); realm->begin_transaction(); realm->commit_transaction(); Object(realm, table->get_object(3)); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); REQUIRE(events.size() == 2); @@ -735,11 +735,11 @@ TEST_CASE("audit object serialization") { list.add(target_table->create_object_with_primary_key(i).set_all(i * 2).get_key()); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); auto obj_list = util::any_cast(object.get_property_value(context, "object list")); obj_list.filter(target_table->where().greater(target_table->get_column_key("value"), 10)).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -780,9 +780,9 @@ TEST_CASE("audit object serialization") { realm->commit_transaction(); SECTION("objects are serialized as just primary key by default") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -795,9 +795,9 @@ TEST_CASE("audit object serialization") { } SECTION("embedded objects are always full object") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -806,10 +806,10 @@ TEST_CASE("audit object serialization") { } SECTION("links followed serialize the full object") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); object.get_property_value(context, "object"); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -825,13 +825,13 @@ TEST_CASE("audit object serialization") { } SECTION("instantiating a collection accessor does not count as a read") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); util::any_cast(object.get_property_value(context, "object list")); util::any_cast(object.get_property_value(context, "object set")); util::any_cast( object.get_property_value(context, "object dictionary")); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -844,7 +844,7 @@ TEST_CASE("audit object serialization") { SECTION("accessing any value from a collection serializes full objects for the entire collection") { SECTION("list") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); auto list = util::any_cast(object.get_property_value(context, "object list")); SECTION("get()") { @@ -853,7 +853,7 @@ TEST_CASE("audit object serialization") { SECTION("get_any()") { list.get_any(1); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -867,7 +867,7 @@ TEST_CASE("audit object serialization") { } SECTION("set") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); auto set = util::any_cast(object.get_property_value(context, "object set")); @@ -877,7 +877,7 @@ TEST_CASE("audit object serialization") { SECTION("get_any()") { set.get_any(1); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -891,7 +891,7 @@ TEST_CASE("audit object serialization") { } SECTION("dictionary") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, obj); auto dict = util::any_cast( object.get_property_value(context, "object dictionary")); @@ -907,7 +907,7 @@ TEST_CASE("audit object serialization") { SECTION("try_get_any()") { dict.try_get_any("b"); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -924,9 +924,9 @@ TEST_CASE("audit object serialization") { SECTION( "link access on an object read outside of a scope does not produce a read on the parent in the scope") { Object object(realm, obj); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); object.get_property_value(context, "object"); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -937,13 +937,13 @@ TEST_CASE("audit object serialization") { } SECTION("link access in a different scope from the object do not expand linked object in parent read") { - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); - scope = audit->begin_scope("scope 2"); + audit->begin_scope("scope 2"); object.get_property_value(context, "object"); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -956,18 +956,18 @@ TEST_CASE("audit object serialization") { } SECTION("link access tracking is reset between scopes") { - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object object(realm, obj); object.get_property_value(context, "object"); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); - scope = audit->begin_scope("scope 2"); + audit->begin_scope("scope 2"); // Perform two unrelated events so that the read on `obj` is at // an event index after the link access in the previous scope Object(realm, target_table->get_object(obj_set.get(0))); Object(realm, target_table->get_object(obj_set.get(1))); Object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -992,10 +992,10 @@ TEST_CASE("audit object serialization") { SECTION("read on the parent after the link access do not expand the linked object") { Object object(realm, obj); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); object.get_property_value(context, "object"); Object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1006,10 +1006,10 @@ TEST_CASE("audit object serialization") { SECTION("read on newly created object") { realm->begin_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object object(realm, table->create_object_with_primary_key(100)); Results(realm, table->where()).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); realm->commit_transaction(); audit->wait_for_completion(); @@ -1024,9 +1024,9 @@ TEST_CASE("audit object serialization") { realm->begin_transaction(); table->create_object_with_primary_key(2); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Results(realm, table->where()).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); realm->commit_transaction(); audit->wait_for_completion(); @@ -1043,12 +1043,12 @@ TEST_CASE("audit object serialization") { realm->commit_transaction(); SECTION("reads of objects that are subsequently deleted are still reported") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); Object(realm, obj2); obj2.remove(); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1059,14 +1059,14 @@ TEST_CASE("audit object serialization") { } SECTION("reads after deletions report the correct object") { - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); obj2.remove(); // In the pre-core-6 version of the code this would incorrectly // report a read on obj2 Object(realm, obj3); realm->commit_transaction(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1098,6 +1098,14 @@ TEST_CASE("audit management") { // but we don't actually want the realm to be synchronizing realm->sync_session()->close(); + SECTION("cannot nest scopes") { + audit->begin_scope("name"); + REQUIRE_THROWS(audit->begin_scope("name")); + } + SECTION("cannot end nonexistent scope") { + REQUIRE_THROWS(audit->end_scope()); + } + SECTION("config validation") { SyncTestFile config(test_session.app(), "parent2"); config.audit_config = std::make_shared(); @@ -1125,13 +1133,13 @@ TEST_CASE("audit management") { auto obj = table->create_object_with_primary_key(1); realm->commit_transaction(); - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); - scope = audit->begin_scope("scope 2"); + audit->begin_scope("scope 2"); Object(realm, obj); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1140,117 +1148,6 @@ TEST_CASE("audit management") { REQUIRE(events[1].activity == "scope 2"); } - SECTION("nested scopes") { - realm->begin_transaction(); - auto obj1 = table->create_object_with_primary_key(1); - auto obj2 = table->create_object_with_primary_key(2); - auto obj3 = table->create_object_with_primary_key(3); - realm->commit_transaction(); - - auto scope1 = audit->begin_scope("scope 1"); - Object(realm, obj1); // read in scope 1 only - - auto scope2 = audit->begin_scope("scope 2"); - Object(realm, obj2); // read in both scopes - audit->end_scope(scope2, assert_no_error); - - Object(realm, obj3); // read in scope 1 only - - audit->end_scope(scope1, assert_no_error); - audit->wait_for_completion(); - - auto events = get_audit_events(test_session); - REQUIRE(events.size() == 4); - - // scope 2 read on obj 2 comes first as it was the first scope ended - REQUIRE(events[0].activity == "scope 2"); - REQUIRE(events[0].data["value"][0]["_id"] == 2); - - // scope 1 then has reads on each object in order - REQUIRE(events[1].activity == "scope 1"); - REQUIRE(events[1].data["value"][0]["_id"] == 1); - REQUIRE(events[2].activity == "scope 1"); - REQUIRE(events[2].data["value"][0]["_id"] == 2); - REQUIRE(events[3].activity == "scope 1"); - REQUIRE(events[3].data["value"][0]["_id"] == 3); - } - - SECTION("overlapping scopes") { - realm->begin_transaction(); - auto obj1 = table->create_object_with_primary_key(1); - auto obj2 = table->create_object_with_primary_key(2); - auto obj3 = table->create_object_with_primary_key(3); - realm->commit_transaction(); - - auto scope1 = audit->begin_scope("scope 1"); - Object(realm, obj1); // read in scope 1 only - - auto scope2 = audit->begin_scope("scope 2"); - Object(realm, obj2); // read in both scopes - - audit->end_scope(scope1, assert_no_error); - Object(realm, obj3); // read in scope 2 only - - audit->end_scope(scope2, assert_no_error); - audit->wait_for_completion(); - - auto events = get_audit_events(test_session); - REQUIRE(events.size() == 4); - - // scope 1 only read on obj 1 - REQUIRE(events[0].activity == "scope 1"); - REQUIRE(events[0].data["value"][0]["_id"] == 1); - - // both scopes read on obj 2 - REQUIRE(events[1].activity == "scope 1"); - REQUIRE(events[1].data["value"][0]["_id"] == 2); - REQUIRE(events[2].activity == "scope 2"); - REQUIRE(events[2].data["value"][0]["_id"] == 2); - - // scope 2 only read on obj 3 - REQUIRE(events[3].activity == "scope 2"); - REQUIRE(events[3].data["value"][0]["_id"] == 3); - } - - SECTION("scope cancellation") { - realm->begin_transaction(); - auto obj = table->create_object_with_primary_key(1); - realm->commit_transaction(); - - auto scope1 = audit->begin_scope("scope 1"); - auto scope2 = audit->begin_scope("scope 2"); - Object(realm, obj); - audit->cancel_scope(scope1); - audit->end_scope(scope2, assert_no_error); - audit->wait_for_completion(); - - auto events = get_audit_events(test_session); - REQUIRE(events.size() == 1); - REQUIRE(events[0].activity == "scope 2"); - } - - SECTION("ending invalid scopes") { - REQUIRE_FALSE(audit->is_scope_valid(0)); - REQUIRE_THROWS_WITH(audit->end_scope(0), - "Cannot end event scope: scope '0' not in progress. Scope may have already been ended?"); - - auto scope = audit->begin_scope("scope"); - REQUIRE(audit->is_scope_valid(scope)); - REQUIRE_NOTHROW(audit->end_scope(scope)); - - REQUIRE_FALSE(audit->is_scope_valid(scope)); - REQUIRE_THROWS_WITH(audit->end_scope(scope), - "Cannot end event scope: scope '1' not in progress. Scope may have already been ended?"); - - scope = audit->begin_scope("scope 2"); - REQUIRE(audit->is_scope_valid(scope)); - REQUIRE_NOTHROW(audit->cancel_scope(scope)); - - REQUIRE_FALSE(audit->is_scope_valid(scope)); - REQUIRE_THROWS_WITH(audit->cancel_scope(scope), - "Cannot end event scope: scope '2' not in progress. Scope may have already been ended?"); - } - SECTION("event timestamps") { std::vector objects; realm->begin_transaction(); @@ -1258,12 +1155,12 @@ TEST_CASE("audit management") { objects.push_back(table->create_object_with_primary_key(i)); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); for (int i = 0; i < 10; ++i) { Object(realm, objects[i]); Object(realm, objects[i]); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1284,9 +1181,9 @@ TEST_CASE("audit management") { SECTION("update before scope") { audit->update_metadata({{"a", "aa"}}); - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object(realm, obj1); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1297,10 +1194,10 @@ TEST_CASE("audit management") { } SECTION("update during scope") { - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); audit->update_metadata({{"a", "aa"}}); Object(realm, obj1); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1312,9 +1209,9 @@ TEST_CASE("audit management") { SECTION("one metadata field at a time") { for (int i = 0; i < 100; ++i) { audit->update_metadata({{util::format("name %1", i), util::format("value %1", i)}}); - auto scope = audit->begin_scope(util::format("scope %1", i)); + audit->begin_scope(util::format("scope %1", i)); Object(realm, obj1); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); } audit->wait_for_completion(); @@ -1331,9 +1228,9 @@ TEST_CASE("audit management") { for (int i = 0; i < 100; ++i) { metadata.push_back({util::format("name %1", i), util::format("value %1", i)}); audit->update_metadata(std::vector(metadata)); - auto scope = audit->begin_scope(util::format("scope %1", i)); + audit->begin_scope(util::format("scope %1", i)); Object(realm, obj1); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); } audit->wait_for_completion(); @@ -1349,20 +1246,20 @@ TEST_CASE("audit management") { auto realm2 = Realm::get_shared_realm(config); auto obj2 = realm2->read_group().get_table("class_object")->get_object(1); - auto scope = audit->begin_scope("scope 1"); + audit->begin_scope("scope 1"); Object(realm, obj1); Object(realm2, obj2); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); config.audit_config->metadata = {{"a", "aaa"}, {"b", "bb"}}; auto realm3 = Realm::get_shared_realm(config); auto obj3 = realm3->read_group().get_table("class_object")->get_object(2); - scope = audit->begin_scope("scope 2"); + audit->begin_scope("scope 2"); Object(realm, obj1); Object(realm2, obj2); Object(realm3, obj3); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1392,10 +1289,10 @@ TEST_CASE("audit management") { audit->record_event("event 1", "event"s, "data"s, expect_completion(0)); audit->record_event("event 2", none, "data"s, expect_completion(1)); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); // note: does not use the scope's activity audit->record_event("event 3", none, none, expect_completion(2)); - audit->end_scope(scope, expect_completion(3)); + audit->end_scope(expect_completion(3)); audit->record_event("event 4", none, none, expect_completion(4)); util::EventLoop::main().run_until([&] { @@ -1436,7 +1333,7 @@ TEST_CASE("audit management") { obj3.set_all(2); realm3->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object(realm3, obj3); // value 2 Object(realm2, obj2); // value 1 Object(realm, obj); // value 0 @@ -1447,7 +1344,7 @@ TEST_CASE("audit management") { Object(realm3, obj3); // value 2 Object(realm2, obj2); // value 2 Object(realm, obj); // value 2 - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1473,12 +1370,12 @@ TEST_CASE("audit management") { auto obj2 = table->create_object_with_primary_key(2); realm->commit_transaction(); - auto scope = audit->begin_scope("large"); + audit->begin_scope("large"); for (int i = 0; i < 150'000; ++i) { Object(realm, obj1); Object(realm, obj2); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); audit->wait_for_completion(); auto events = get_audit_events(test_session); @@ -1525,9 +1422,9 @@ TEST_CASE("audit realm sharding") { // Write a lot of audit scopes while unable to sync for (int i = 0; i < 50; ++i) { - auto scope = audit->begin_scope(util::format("scope %1", i)); + audit->begin_scope(util::format("scope %1", i)); Results(realm, table->where()).snapshot(); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); } audit->wait_for_completion(); @@ -1653,9 +1550,9 @@ static void generate_event(std::shared_ptr realm, int call = 0) table->create_object_with_primary_key(call + 1).set_all(2); realm->commit_transaction(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); Object(realm, table->get_object(call)); - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); } TEST_CASE("audit integration tests") { @@ -1785,13 +1682,13 @@ TEST_CASE("audit integration tests") { session.app()->sync_manager()->remove_user(audit_user->identity()); auto audit = realm->audit_context(); - auto scope = audit->begin_scope("scope"); + audit->begin_scope("scope"); realm->begin_transaction(); auto table = realm->read_group().get_table("class_object"); table->create_object_with_primary_key(1).set_all(2); realm->commit_transaction(); - audit->end_scope(scope, [&](auto error) { + audit->end_scope([&](auto error) { REQUIRE(error); REQUIRE_THROWS_CONTAINING(std::rethrow_exception(error), "user has been removed"); }); @@ -1881,9 +1778,8 @@ TEST_CASE("audit integration tests") { SECTION("auditing with a flexible sync user reports a sync error") { config.audit_config->audit_user = harness.app()->current_user(); auto error = expect_error(config, generate_event); - REQUIRE_THAT(error.message, - Catch::Matchers::ContainsSubstring( - "Client connected using partition-based sync when app is using flexible sync")); + REQUIRE(error.message.find( + "client connected using partition based sync when app is using flexible sync") == 0); REQUIRE(error.is_fatal); } @@ -1900,7 +1796,7 @@ TEST_CASE("audit integration tests") { std::move(mut_subs).commit(); } - realm->sync_session()->force_close(); + realm->sync_session()->log_out(); generate_event(realm, 0); get_audit_events_from_baas(session, *config.audit_config->audit_user, 1); } @@ -1917,12 +1813,12 @@ TEST_CASE("audit integration tests") { auto obj2 = table->create_object_with_primary_key(2); realm->commit_transaction(); - auto scope = audit->begin_scope("large"); + audit->begin_scope("large"); for (int i = 0; i < 150'000; ++i) { Object(realm, obj1); Object(realm, obj2); } - audit->end_scope(scope, assert_no_error); + audit->end_scope(assert_no_error); REQUIRE(get_audit_events_from_baas(session, *session.app()->current_user(), 300'000).size() == 300'000); } diff --git a/test/object-store/c_api/c_api.cpp b/test/object-store/c_api/c_api.cpp index a2b85c4bbc9..9d3a1206539 100644 --- a/test/object-store/c_api/c_api.cpp +++ b/test/object-store/c_api/c_api.cpp @@ -25,7 +25,6 @@ #if REALM_ENABLE_AUTH_TESTS #include #include -#include #include #include #include "sync/sync_test_utils.hpp" @@ -122,18 +121,6 @@ realm_value_t rlm_decimal_val(double d) return val; } -// realm_value_t rlm_decimal_nan() -// { -// realm_value_t val; -// val.type = RLM_TYPE_DECIMAL128; - -// realm::Decimal128 dec = realm::Decimal128::nan("0"); -// val.decimal128.w[0] = dec.raw()->w[0]; -// val.decimal128.w[1] = dec.raw()->w[1]; - -// return val; -// } - realm_value_t rlm_uuid_val(const char* str) { realm_value_t val; @@ -638,7 +625,7 @@ TEST_CASE("C API (non-database)", "[c_api]") { }); } - SECTION("realm_sync_error_code") { + SECTION("realm_sync_error_code to_capi()") { using namespace realm::sync; std::string message; @@ -649,51 +636,25 @@ TEST_CASE("C API (non-database)", "[c_api]") { CHECK(error_code.message() == error.message); CHECK(message == error.message); - std::error_code ec_check; - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == realm::sync::client_error_category()); - CHECK(ec_check.value() == int(error_code.value())); - error_code = make_error_code(sync::ProtocolError::connection_closed); error = c_api::to_capi(error_code, message); CHECK(error.category == realm_sync_error_category_e::RLM_SYNC_ERROR_CATEGORY_CONNECTION); - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == realm::sync::protocol_error_category()); - CHECK(ec_check.value() == int(error_code.value())); - error_code = make_error_code(sync::ProtocolError::session_closed); error = c_api::to_capi(error_code, message); CHECK(error.category == realm_sync_error_category_e::RLM_SYNC_ERROR_CATEGORY_SESSION); - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == realm::sync::protocol_error_category()); - CHECK(ec_check.value() == int(error_code.value())); - error_code = make_error_code(realm::util::error::basic_system_errors::invalid_argument); error = c_api::to_capi(error_code, message); CHECK(error.category == realm_sync_error_category_e::RLM_SYNC_ERROR_CATEGORY_SYSTEM); - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == std::system_category()); - CHECK(ec_check.value() == int(error_code.value())); - - error_code = make_error_code(sync::network::ResolveErrors::socket_type_not_supported); + error_code = make_error_code(sync::network::ResolveErrors::host_not_found); error = c_api::to_capi(error_code, message); CHECK(error.category == realm_sync_error_category_e::RLM_SYNC_ERROR_CATEGORY_RESOLVE); - CHECK(error.value == realm_sync_error_resolve_e::RLM_SYNC_ERROR_RESOLVE_SOCKET_TYPE_NOT_SUPPORTED); - - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == realm::sync::network::resolve_error_category()); - CHECK(ec_check.value() == int(error_code.value())); error_code = make_error_code(util::error::misc_errors::unknown); error = c_api::to_capi(error_code, message); CHECK(error.category == realm_sync_error_category_e::RLM_SYNC_ERROR_CATEGORY_UNKNOWN); - - c_api::sync_error_to_error_code(error, &ec_check); - CHECK(ec_check.category() == realm::util::error::basic_system_error_category()); - CHECK(ec_check.value() == int(error_code.value())); } @@ -2282,27 +2243,6 @@ TEST_CASE("C API", "[c_api]") { CHECK_ERR(RLM_ERR_INDEX_OUT_OF_BOUNDS); } - SECTION("decimal NaN") { - // TODO: re-enable and fix this test before to merge into master - // realm_value_t decimal = rlm_decimal_nan(); - - // write([&]() { - // CHECK(realm_set_value(obj1.get(), foo_properties["decimal"], decimal, false)); - // }); - // realm_query_arg_t args[] = {realm_query_arg_t{1, false, &decimal}}; - // auto q_decimal = cptr_checked(realm_query_parse(realm, class_foo.key, "decimal == $0", 1, args)); - // realm_value_t out_value; - // bool out_found; - // CHECK(realm_query_find_first(q_decimal.get(), &out_value, &out_found)); - // CHECK(out_found); - // auto link = obj1->obj().get_link(); - // realm_value_t expected; - // expected.type = RLM_TYPE_LINK; - // expected.link.target_table = link.get_table_key().value; - // expected.link.target = link.get_obj_key().value; - // CHECK(rlm_val_eq(out_value, expected)); - } - SECTION("interpolate all types") { realm_value_t int_arg = rlm_int_val(123); realm_value_t bool_arg = rlm_bool_val(true); @@ -3305,13 +3245,11 @@ TEST_CASE("C API", "[c_api]") { CHECK(num_moves == 0); size_t num_deletions, num_insertions, num_modifications; - bool collection_cleared = false; realm_collection_changes_get_num_changes(state.changes.get(), &num_deletions, &num_insertions, - &num_modifications, &num_moves, &collection_cleared); + &num_modifications, &num_moves); CHECK(num_deletions == 1); CHECK(num_insertions == 2); CHECK(num_modifications == 1); - CHECK(collection_cleared == false); realm_index_range_t deletions, insertions, modifications, modifications_after; realm_collection_move_t moves; @@ -3348,14 +3286,6 @@ TEST_CASE("C API", "[c_api]") { CHECK(modifications_v[1] == size_t(-1)); CHECK(modifications_after_v[0] == 2); CHECK(modifications_after_v[1] == size_t(-1)); - - write([&]() { - checked(realm_list_clear(strings.get())); - }); - - realm_collection_changes_get_num_changes(state.changes.get(), &num_deletions, &num_insertions, - &num_modifications, &num_moves, &collection_cleared); - CHECK(collection_cleared == true); } } } @@ -3774,16 +3704,6 @@ TEST_CASE("C API", "[c_api]") { CHECK(deletion_range.to == 1); CHECK(insertion_range.from == 0); CHECK(insertion_range.to == 2); - - write([&]() { - checked(realm_set_clear(strings.get())); - }); - - size_t num_deletions, num_insertions, num_modifications; - bool collection_cleared = false; - realm_collection_changes_get_num_changes(state.changes.get(), &num_deletions, &num_insertions, - &num_modifications, &num_moves, &collection_cleared); - CHECK(collection_cleared == true); } } } @@ -4234,15 +4154,15 @@ TEST_CASE("C API", "[c_api]") { SECTION("notifications") { struct State { CPtr changes; - CPtr dictionary_changes; CPtr error; bool destroyed = false; }; State state; - auto on_dictionary_change = [](void* userdata, const realm_dictionary_changes_t* changes) { + + auto on_change = [](void* userdata, const realm_collection_changes_t* changes) { auto* state = static_cast(userdata); - state->dictionary_changes = clone_cptr(changes); + state->changes = clone_cptr(changes); }; CPtr strings = @@ -4254,7 +4174,7 @@ TEST_CASE("C API", "[c_api]") { auto require_change = [&]() { auto token = cptr_checked(realm_dictionary_add_notification_callback( - strings.get(), &state, nullptr, nullptr, on_dictionary_change)); + strings.get(), &state, nullptr, nullptr, on_change)); checked(realm_refresh(realm, nullptr)); return token; }; @@ -4282,75 +4202,24 @@ TEST_CASE("C API", "[c_api]") { checked(realm_dictionary_insert(strings.get(), rlm_str_val("c"), null, nullptr, nullptr)); }); CHECK(!state.error); - CHECK(state.dictionary_changes); - - size_t num_deletions, num_insertions, num_modifications; - realm_dictionary_get_changes(state.dictionary_changes.get(), &num_deletions, &num_insertions, - &num_modifications); - CHECK(num_deletions == 1); - CHECK(num_insertions == 2); - CHECK(num_modifications == 0); - realm_value_t *deletions = nullptr, *insertions = nullptr, *modifications = nullptr; - deletions = (realm_value_t*)malloc(sizeof(realm_value_t) * num_deletions); - insertions = (realm_value_t*)malloc(sizeof(realm_value_t) * num_insertions); - realm_dictionary_get_changed_keys(state.dictionary_changes.get(), deletions, &num_deletions, - insertions, &num_insertions, modifications, &num_modifications); - CHECK(deletions != nullptr); - CHECK(insertions != nullptr); - CHECK(modifications == nullptr); - realm_free(deletions); - realm_free(insertions); - realm_free(modifications); - } - } - - SECTION("realm_dictionary_content_checks") { - auto ints = cptr_checked(realm_get_dictionary(obj1.get(), foo_properties["int_dict"])); - CHECK(ints); - CHECK(!realm_is_frozen(ints.get())); - realm_value_t key1 = rlm_str_val("k"); - realm_value_t key2 = rlm_str_val("k2"); - realm_value_t integer1 = rlm_int_val(987); - realm_value_t integer2 = rlm_int_val(988); - - write([&]() { - bool inserted = false; - CHECK(checked(realm_dictionary_insert(ints.get(), key1, integer1, nullptr, &inserted))); - CHECK(inserted); - CHECK(checked(realm_dictionary_insert(ints.get(), key2, integer2, nullptr, &inserted))); - CHECK(inserted); - }); - - SECTION("realm_dictionary_get_keys") { - size_t size = 0; - realm_results_t* keys = nullptr; - CHECK(checked(realm_dictionary_get_keys(ints.get(), &size, &keys))); - CHECK(keys); - CHECK((*keys).size() == size); - realm_release(keys); - } + CHECK(state.changes); - SECTION("realm_dictionary_contains_key") { - bool found = false; - CHECK(checked(realm_dictionary_contains_key(ints.get(), key1, &found))); - CHECK(found); - found = false; - CHECK(checked(realm_dictionary_contains_key(ints.get(), key2, &found))); - CHECK(found); - realm_value_t key_no_present = rlm_str_val("kkkk"); - CHECK(checked(realm_dictionary_contains_key(ints.get(), key_no_present, &found))); - CHECK(!found); - } + size_t num_deletion_ranges, num_insertion_ranges, num_modification_ranges, num_moves; + realm_collection_changes_get_num_ranges(state.changes.get(), &num_deletion_ranges, + &num_insertion_ranges, &num_modification_ranges, + &num_moves); + CHECK(num_deletion_ranges == 1); + CHECK(num_insertion_ranges == 1); + CHECK(num_modification_ranges == 0); + CHECK(num_moves == 0); - SECTION("realm_dictionary_contains_value") { - size_t index = -1; - CHECK(checked(realm_dictionary_contains_value(ints.get(), integer1, &index))); - CHECK(index == 0); - CHECK(checked(realm_dictionary_contains_value(ints.get(), integer2, &index))); - CHECK(index == 1); - realm_value_t integer_no_present = rlm_int_val(678); - CHECK(checked(realm_dictionary_contains_value(ints.get(), integer_no_present, &index))); - CHECK(index == realm::npos); + realm_index_range_t deletion_range, insertion_range; + realm_collection_changes_get_ranges(state.changes.get(), &deletion_range, 1, &insertion_range, 1, + nullptr, 0, nullptr, 0, nullptr, 0); + CHECK(deletion_range.from == 0); + CHECK(deletion_range.to == 1); + CHECK(insertion_range.from == 0); + CHECK(insertion_range.to == 2); } } } @@ -5803,163 +5672,4 @@ TEST_CASE("app: flx-sync basic tests", "[c_api][flx][sync]") { realm_release(c_wrap_query_bar); }); } - -TEST_CASE("C API app: websocket provider", "[c_api][sync][app]") { - using namespace realm::app; - using namespace realm::sync; - using namespace realm::sync::websocket; - - struct TestWebSocketObserverShim : sync::WebSocketObserver { - public: - explicit TestWebSocketObserverShim(std::shared_ptr observer) - : m_observer(observer) - { - } - - void websocket_connected_handler(const std::string& protocol) override - { - return m_observer->websocket_connected_handler(protocol); - } - - void websocket_error_handler() override - { - m_observer->websocket_error_handler(); - } - - bool websocket_binary_message_received(util::Span data) override - { - return m_observer->websocket_binary_message_received(data); - } - - bool websocket_closed_handler(bool was_clean, Status status) override - { - return m_observer->websocket_closed_handler(was_clean, std::move(status)); - } - - private: - std::shared_ptr m_observer; - }; - - struct TestWebSocket : realm::c_api::WrapC, WebSocketInterface { - public: - TestWebSocket(DefaultSocketProvider& socket_provider, realm_websocket_endpoint_t endpoint, - realm_websocket_observer_t* realm_websocket_observer) - { - WebSocketEndpoint ws_endpoint; - ws_endpoint.address = endpoint.address; - ws_endpoint.port = endpoint.port; - ws_endpoint.path = endpoint.path; - for (size_t i = 0; i < endpoint.num_protocols; ++i) { - ws_endpoint.protocols.push_back(endpoint.protocols[i]); - } - ws_endpoint.is_ssl = endpoint.is_ssl; - - auto observer = std::make_unique(*realm_websocket_observer); - m_websocket = socket_provider.connect(std::move(observer), std::move(ws_endpoint)); - } - - void async_write_binary(util::Span data, SyncSocketProvider::FunctionHandler&& handler) override - { - m_websocket->async_write_binary(data, std::move(handler)); - } - - private: - std::unique_ptr m_websocket; - }; - - struct TestSyncTimer : realm::c_api::WrapC, SyncSocketProvider::Timer { - public: - TestSyncTimer(DefaultSocketProvider& socket_provider, std::chrono::milliseconds delay, - SyncSocketProvider::FunctionHandler&& handler) - { - m_timer = socket_provider.create_timer(delay, std::move(handler)); - } - - void cancel() override - { - m_timer->cancel(); - } - - private: - SyncSocketProvider::SyncTimer m_timer; - }; - - struct TestData { - DefaultSocketProvider* socket_provider; - int free_count = 0; - }; - - auto logger = std::make_shared(); - DefaultSocketProvider default_socket_provider(logger, "SocketProvider"); - - auto free_fn = [](realm_userdata_t user_ptr) { - auto test_data = static_cast(user_ptr); - REQUIRE(test_data); - test_data->free_count++; - }; - auto post_fn = [](realm_userdata_t userdata, realm_sync_socket_callback_t* callback) { - auto test_data = static_cast(userdata); - REQUIRE(test_data); - auto cb = [callback_copy = callback](Status s) { - realm_sync_socket_callback_complete(callback_copy, static_cast(s.code()), - s.reason().c_str()); - }; - test_data->socket_provider->post(std::move(cb)); - }; - auto create_timer_fn = [](realm_userdata_t userdata, uint64_t delay_ms, - realm_sync_socket_callback_t* callback) -> realm_sync_socket_timer_t { - auto test_data = static_cast(userdata); - REQUIRE(test_data); - return static_cast(new TestSyncTimer( - *test_data->socket_provider, std::chrono::milliseconds(delay_ms), std::move(**callback))); - }; - auto cancel_timer_fn = [](realm_userdata_t, realm_sync_socket_timer_t sync_timer) { - auto timer = static_cast(sync_timer); - REQUIRE(timer); - timer->cancel(); - }; - auto free_timer_fn = [](realm_userdata_t, realm_sync_socket_timer_t sync_timer) { - realm_release(sync_timer); - }; - auto websocket_connect_fn = - [](realm_userdata_t userdata, realm_websocket_endpoint_t endpoint, - realm_websocket_observer_t* realm_websocket_observer) -> realm_sync_socket_websocket_t { - auto test_data = static_cast(userdata); - REQUIRE(test_data); - return static_cast( - new TestWebSocket(*test_data->socket_provider, endpoint, realm_websocket_observer)); - }; - auto websocket_async_write_fn = [](realm_userdata_t, realm_sync_socket_websocket_t sync_websocket, - const char* data, size_t size, realm_sync_socket_callback_t* callback) { - auto websocket = static_cast(sync_websocket); - REQUIRE(websocket); - websocket->async_write_binary(util::Span{data, size}, std::move(**callback)); - realm_release(callback); - }; - auto websocket_free_fn = [](realm_userdata_t, realm_sync_socket_websocket_t sync_websocket) { - realm_release(sync_websocket); - }; - - // Test drive. - TestData test_data{&default_socket_provider}; - { - auto socket_provider = realm_sync_socket_new( - static_cast(&test_data), free_fn, post_fn, create_timer_fn, cancel_timer_fn, - free_timer_fn, websocket_connect_fn, websocket_async_write_fn, websocket_free_fn); - - - FLXSyncTestHarness harness("c_api_websocket_provider", FLXSyncTestHarness::default_server_schema(), - instance_of, *socket_provider); - - SyncTestFile test_config(harness.app()->current_user(), harness.schema(), - realm::SyncConfig::FLXSyncEnabled{}); - auto realm = Realm::get_shared_realm(test_config); - REQUIRE(!wait_for_download(*realm)); - - realm_release(socket_provider); - } - - default_socket_provider.stop(true); - REQUIRE(test_data.free_count == 1); -} #endif // REALM_ENABLE_AUTH_TESTS diff --git a/test/object-store/collection_fixtures.hpp b/test/object-store/collection_fixtures.hpp index b5279af2d37..155caebfaf7 100644 --- a/test/object-store/collection_fixtures.hpp +++ b/test/object-store/collection_fixtures.hpp @@ -288,36 +288,9 @@ struct MixedVal : Base { enum { is_optional = true }; static std::vector values() { - return { - Mixed{realm::UUID()}, - Mixed{}, - Mixed{realm::ObjectId()}, - - // Mixed sorting considers all numerics to be the same time, so - // ensure we have some interleaved values to test that - Mixed{int64_t(1)}, - Mixed{int64_t(2)}, - Mixed{int64_t(3)}, - - Mixed{double(1.2)}, - Mixed{double(2.2)}, - Mixed{double(3.2)}, - - Mixed{float(1.1)}, - Mixed{float(2.1)}, - Mixed{float(3.1)}, - - Mixed{Decimal128("1.3")}, - Mixed{Decimal128("2.3")}, - Mixed{Decimal128("3.3")}, - - // Mixed sorting considers strings and binary to be the same time, so - // ensure we have some interleaved values to test that - Mixed{"a string"}, - Mixed{"b string"}, - Mixed{BinaryData("a binary", 8)}, - Mixed{BinaryData("b binary", 8)}, - }; + return {Mixed{realm::UUID()}, Mixed{int64_t(1)}, Mixed{}, + Mixed{"hello world"}, Mixed{Timestamp(1, 1)}, Mixed{Decimal128("300")}, + Mixed{double(2.2)}, Mixed{float(3.3)}}; } static PropertyType property_type() { @@ -333,13 +306,11 @@ struct MixedVal : Base { } static Decimal128 sum() { - return Decimal128{int64_t(1)} + Decimal128{int64_t(2)} + Decimal128{int64_t(3)} + Decimal128{double(1.2)} + - Decimal128{double(2.2)} + Decimal128{double(3.2)} + Decimal128{float(1.1)} + Decimal128{float(2.1)} + - Decimal128{float(3.1)} + Decimal128("1.3") + Decimal128("2.3") + Decimal128("3.3"); + return Decimal128("300") + Decimal128(int64_t(1)) + Decimal128(double(2.2)) + Decimal128(float(3.3)); } static Decimal128 average() { - return (sum() / Decimal128(12)); + return (sum() / Decimal128(4)); } static Mixed empty_sum_value() { diff --git a/test/object-store/dictionary.cpp b/test/object-store/dictionary.cpp index b1e7a54375e..051e1da9fe3 100644 --- a/test/object-store/dictionary.cpp +++ b/test/object-store/dictionary.cpp @@ -937,8 +937,7 @@ TEMPLATE_TEST_CASE("dictionary of objects", "[dictionary][links]", cf::MixedVal, Obj target_obj = target->create_object().set(col_target_value, T(values[i])); dict.insert(keys[i], target_obj); } - r->commit_transaction(); - r->begin_transaction(); + SECTION("min()") { if (!TestType::can_minmax()) { REQUIRE_THROWS_AS(dict.min(col_target_value), Results::UnsupportedColumnTypeException); @@ -1348,85 +1347,3 @@ TEST_CASE("dictionary aggregate", "[dictionary]") { auto sum = res.sum("intCol"); REQUIRE(*sum == 16); } - -TEST_CASE("callback with empty keypatharray") { - InMemoryTestFile config; - config.schema = Schema{ - {"object", {{"links", PropertyType::Dictionary | PropertyType::Object | PropertyType::Nullable, "target"}}}, - {"target", {{"value", PropertyType::Int}}}, - }; - - auto r = Realm::get_shared_realm(config); - auto table = r->read_group().get_table("class_object"); - auto target = r->read_group().get_table("class_target"); - - r->begin_transaction(); - Obj obj = table->create_object(); - ColKey col_links = table->get_column_key("links"); - ColKey col_target_value = target->get_column_key("value"); - object_store::Dictionary dict(r, obj, col_links); - auto key = "key"; - Obj target_obj = target->create_object().set(col_target_value, 1); - dict.insert(key, target_obj); - r->commit_transaction(); - - CollectionChangeSet change; - auto write = [&](auto&& f) { - r->begin_transaction(); - f(); - r->commit_transaction(); - advance_and_notify(*r); - }; - - auto shallow_require_change = [&] { - auto token = dict.add_notification_callback( - [&](CollectionChangeSet c) { - change = c; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - auto shallow_require_no_change = [&] { - bool first = true; - auto token = dict.add_notification_callback( - [&first](CollectionChangeSet) mutable { - REQUIRE(first); - first = false; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - SECTION("insertion DOES send notification") { - auto token = shallow_require_change(); - write([&] { - Obj target_obj = target->create_object().set(col_target_value, 1); - dict.insert("foo", target_obj); - }); - REQUIRE_FALSE(change.insertions.empty()); - } - SECTION("deletion DOES send notification") { - auto token = shallow_require_change(); - write([&] { - dict.erase(key); - }); - REQUIRE_FALSE(change.deletions.empty()); - } - SECTION("replacement DOES send notification") { - auto token = shallow_require_change(); - write([&] { - Obj target_obj = target->create_object().set(col_target_value, 1); - dict.insert(key, target_obj); - }); - REQUIRE_FALSE(change.modifications.empty()); - } - SECTION("modification does NOT send notification") { - auto token = shallow_require_no_change(); - write([&] { - dict.get(key).set(col_target_value, 2); - }); - } -} diff --git a/test/object-store/list.cpp b/test/object-store/list.cpp index 1131d9ac3dd..b9a7a6bf854 100644 --- a/test/object-store/list.cpp +++ b/test/object-store/list.cpp @@ -712,7 +712,6 @@ TEST_CASE("list") { // - some callbacks have filters // - all callbacks have filters CollectionChangeSet collection_change_set_without_filter; - CollectionChangeSet collection_change_set_with_empty_filter; CollectionChangeSet collection_change_set_with_filter_on_target_value; // Note that in case not all callbacks have filters we do accept false positive notifications by design. @@ -808,73 +807,6 @@ TEST_CASE("list") { } } - SECTION("callback with empty keypatharray") { - auto shallow_require_change = [&] { - auto token = list.add_notification_callback( - [&](CollectionChangeSet c) { - collection_change_set_with_empty_filter = c; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - auto shallow_require_no_change = [&] { - bool first = true; - auto token = list.add_notification_callback( - [&first](CollectionChangeSet) mutable { - REQUIRE(first); - first = false; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - SECTION("modifying table 'target', property 'value' " - "-> does NOT send a notification for 'value'") { - auto token = shallow_require_no_change(); - write([&] { - list.get(0).set(col_target_value, 42); - }); - } - - SECTION("modifying table 'target', property 'value' " - "-> does NOT send a notification for 'value'") { - auto token = shallow_require_no_change(); - write([&] { - list.get(0).set(col_target_value, 42); - }); - } - - SECTION("deleting a target row with shallow listener sends a change notification") { - auto token = shallow_require_change(); - write([&] { - list.remove(5); - }); - REQUIRE_INDICES(collection_change_set_with_empty_filter.deletions, 5); - } - - SECTION("adding a row with shallow listener sends a change notifcation") { - auto token = shallow_require_change(); - write([&] { - Obj obj = target->get_object(target_keys[5]); - list.add(obj); - }); - REQUIRE_INDICES(collection_change_set_with_empty_filter.insertions, 10); - } - - SECTION("moving a row with shallow listener sends a change notifcation") { - auto token = shallow_require_change(); - write([&] { - list.move(5, 8); - }); - REQUIRE_INDICES(collection_change_set_with_empty_filter.insertions, 8); - REQUIRE_INDICES(collection_change_set_with_empty_filter.deletions, 5); - REQUIRE_MOVES(collection_change_set_with_empty_filter, {5, 8}); - } - } - SECTION("linked filter") { CollectionChangeSet collection_change_set_linked_filter; Object object(r, obj); diff --git a/test/object-store/migrations.cpp b/test/object-store/migrations.cpp index cacf74f364a..0d622d69bd9 100644 --- a/test/object-store/migrations.cpp +++ b/test/object-store/migrations.cpp @@ -211,6 +211,199 @@ auto create_objects(Table& table, size_t count) } } // anonymous namespace +TEST_CASE("migration: Additive mode returns OS schema - Automatic migration") { + + SECTION("Check OS schema returned in additive mode") { + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.schema_mode = SchemaMode::AdditiveExplicit; + auto realm = Realm::get_shared_realm(config); + + auto update_schema = [](Realm& r, Schema& s, uint64_t version) { + REQUIRE_NOTHROW((r).update_schema(s, version)); + VERIFY_SCHEMA(r, false); + auto schema = (r).schema(); + for (const auto& other : s) { + REQUIRE(schema.find(other.name) != schema.end()); + } + }; + + Schema schema1 = {}; + Schema schema2 = add_table(schema1, {"A", {{"value", PropertyType::Int}}}); + Schema schema3 = add_table(schema2, {"B", {{"value", PropertyType::Int}}}); + Schema schema4 = add_table(schema3, {"C", {{"value", PropertyType::Int}}}); + Schema schema5 = add_table(schema4, {"Z", {{"value", PropertyType::Int}}}); + update_schema(*realm, schema1, 0); + REQUIRE(realm->schema().size() == 0); + update_schema(*realm, schema2, 0); + REQUIRE(realm->schema().size() == 1); + update_schema(*realm, schema3, 0); + REQUIRE(realm->schema().size() == 2); + update_schema(*realm, schema4, 0); + REQUIRE(realm->schema().size() == 3); + update_schema(*realm, schema5, 0); + REQUIRE(realm->schema().size() == 4); + + // schema size is decremented. + // after deletion the schema size is decremented but the just deleted object can still be found. + // the object that was just deleted is still there, thus find should return a valid iterator + SECTION("delete in reverse order") { + auto new_schema = schema5; + Schema delete_schema = remove_table(new_schema, "Z"); + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("Z") != schema.end()); + delete_schema = remove_table(schema4, "C"); + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + delete_schema = remove_table(schema3, "B"); + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + delete_schema = remove_table(schema2, "A"); + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + SECTION("delete 1 element") { + auto new_schema = schema5; + Schema delete_schema = remove_table(new_schema, "Z"); + // A B C Z vs A B C ==> Z (other classes) + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + delete_schema = remove_table(new_schema, "C"); + schema = realm->schema(); + // A B C vs A B Z => Z + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + delete_schema = remove_table(new_schema, "B"); + // A B Z vs A C Z => B + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + delete_schema = remove_table(new_schema, "A"); + // A B Z vs B C Z => A + update_schema(*realm, delete_schema, 0); + schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + SECTION("delete 2 elements") { + auto new_schema = schema5; + Schema delete_schema; + delete_schema = remove_table(new_schema, "Z"); + delete_schema = remove_table(delete_schema, "A"); + // A B C Z vs B C ==> A,Z (other classes) + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + SECTION("delete 3 elements") { + auto new_schema = schema5; + Schema delete_schema; + delete_schema = remove_table(new_schema, "Z"); + delete_schema = remove_table(delete_schema, "A"); + delete_schema = remove_table(delete_schema, "C"); + // A B C Z vs B ==> A,C,Z (other classes) + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + SECTION("delete all elements") { + auto new_schema = schema5; + Schema delete_schema; + delete_schema = remove_table(new_schema, "Z"); + delete_schema = remove_table(delete_schema, "A"); + delete_schema = remove_table(delete_schema, "C"); + delete_schema = remove_table(delete_schema, "B"); + // A B C Z vs None ==> A,C,Z,B (other classes) + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + SECTION("unsorted schema object names") { + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.schema_mode = SchemaMode::AdditiveExplicit; + auto realm = Realm::get_shared_realm(config); + + Schema schema1 = {}; + Schema schema2 = add_table(schema1, {"Z", {{"value", PropertyType::Int}}}); + Schema schema3 = add_table(schema2, {"B", {{"value", PropertyType::Int}}}); + Schema schema4 = add_table(schema3, {"A", {{"value", PropertyType::Int}}}); + Schema schema5 = add_table(schema4, {"C", {{"value", PropertyType::Int}}}); + update_schema(*realm, schema5, 0); + + Schema delete_schema; + delete_schema = remove_table(schema5, "Z"); + delete_schema = remove_table(delete_schema, "A"); + // Z B A C vs Z A => B C (others) + update_schema(*realm, delete_schema, 0); + auto schema = realm->schema(); + REQUIRE(schema.size() == 4); + REQUIRE(schema.find("C") != schema.end()); + REQUIRE(schema.find("Z") != schema.end()); + REQUIRE(schema.find("A") != schema.end()); + REQUIRE(schema.find("B") != schema.end()); + } + + SECTION("frozen realm schema can be updated if it is a subset of OS schema") { + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.schema_mode = SchemaMode::AdditiveExplicit; + auto realm = Realm::get_shared_realm(config); + Schema schema1 = {}; + Schema schema2 = add_table(schema1, {"Z", {{"value", PropertyType::Int}}}); + Schema schema3 = add_table(schema2, {"B", {{"value", PropertyType::Int}}}); + Schema schema4 = add_table(schema3, {"A", {{"value", PropertyType::Int}}}); + update_schema(*realm, schema4, 0); + auto frozen_realm = realm->freeze(); + auto subset_schema = remove_table(schema4, "Z"); + update_schema(*realm, subset_schema, 0); + REQUIRE_NOTHROW(frozen_realm->update_schema(realm->schema())); + } + } +} + TEST_CASE("migration: Automatic") { InMemoryTestFile config; config.automatic_change_notifications = false; @@ -2112,15 +2305,30 @@ TEST_CASE("migration: SoftResetFile") { {"object 2", {{"value", PropertyType::Int}}}, }; - auto get_fileid = [&] { - auto id = util::File::get_unique_id(config.path); - REQUIRE(id); - return *id; - }; // To verify that the file has actually be deleted and recreated, on // non-Windows we need to hold an open file handle to the old file to force // using a new inode, but on Windows we *can't* -#ifndef _WIN32 +#ifdef _WIN32 + auto get_fileid = [&] { + // this is wrong for non-ascii but it's what core does + std::wstring ws(config.path.begin(), config.path.end()); + HANDLE handle = + CreateFile2(ws.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, nullptr); + REQUIRE(handle != INVALID_HANDLE_VALUE); + auto close = util::make_scope_exit([=]() noexcept { + CloseHandle(handle); + }); + + BY_HANDLE_FILE_INFORMATION info{}; + REQUIRE(GetFileInformationByHandle(handle, &info)); + return (DWORDLONG)info.nFileIndexHigh + (DWORDLONG)info.nFileIndexLow; + }; +#else + auto get_fileid = [&] { + util::File::UniqueID id; + util::File::get_unique_id(config.path, id); + return id.inode; + }; util::File holder(config.path, util::File::mode_Write); #endif @@ -2185,12 +2393,30 @@ TEST_CASE("migration: HardResetFile") { {"object 2", {{"value", PropertyType::Int}}}, }; +// To verify that the file has actually be deleted and recreated, on +// non-Windows we need to hold an open file handle to the old file to force +// using a new inode, but on Windows we *can't* +#ifdef _WIN32 + auto get_fileid = [&] { + // this is wrong for non-ascii but it's what core does + std::wstring ws(config.path.begin(), config.path.end()); + HANDLE handle = + CreateFile2(ws.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, nullptr); + REQUIRE(handle != INVALID_HANDLE_VALUE); + auto close = util::make_scope_exit([=]() noexcept { + CloseHandle(handle); + }); + + BY_HANDLE_FILE_INFORMATION info{}; + REQUIRE(GetFileInformationByHandle(handle, &info)); + return (DWORDLONG)info.nFileIndexHigh + (DWORDLONG)info.nFileIndexLow; + }; +#else auto get_fileid = [&] { - auto id = util::File::get_unique_id(config.path); - REQUIRE(id); - return *id; + util::File::UniqueID id; + util::File::get_unique_id(config.path, id); + return id.inode; }; -#ifndef _WIN32 util::File holder(config.path, util::File::mode_Write); #endif @@ -2311,7 +2537,7 @@ TEST_CASE("migration: Additive") { REQUIRE_NOTHROW(realm->update_schema(remove_property(schema, "object", "value"))); REQUIRE(ObjectStore::table_for_object_type(realm->read_group(), "object")->get_column_count() == 2); auto const& properties = realm->schema().find("object")->persisted_properties; - REQUIRE(properties.size() == 1); + REQUIRE(properties.size() == 2); auto col_keys = table->get_column_keys(); REQUIRE(col_keys.size() == 2); REQUIRE(properties[0].column_key == col_keys[1]); @@ -2366,7 +2592,10 @@ TEST_CASE("migration: Additive") { })); } - SECTION("new columns added externally are ignored") { + SECTION("add new columns from different SG") { + using vec = std::vector; + using namespace schema_change; + auto realm2 = Realm::get_shared_realm(config); auto& group = realm2->read_group(); realm2->begin_transaction(); @@ -2376,17 +2605,19 @@ TEST_CASE("migration: Additive") { realm2->commit_transaction(); REQUIRE_NOTHROW(realm->refresh()); - REQUIRE(realm->schema() == schema); + auto schema_diff = schema.compare(realm->schema()); + REQUIRE(schema_diff.size() == 1); + REQUIRE(schema_diff == vec{(AddProperty{&*schema.find("object"), + &realm->schema().find("object")->persisted_properties[2]})}); REQUIRE(realm->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); REQUIRE(realm->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); - - auto frozen = realm->freeze(); - REQUIRE(frozen->schema() == schema); - REQUIRE(frozen->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); - REQUIRE(frozen->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); + REQUIRE(realm->schema().find("object")->persisted_properties[2].column_key == col_keys[2]); } SECTION("opening new Realms uses the correct schema after an external change") { + using vec = std::vector; + using namespace schema_change; + auto realm2 = Realm::get_shared_realm(config); auto& group = realm2->read_group(); realm2->begin_transaction(); @@ -2396,9 +2627,13 @@ TEST_CASE("migration: Additive") { realm2->commit_transaction(); REQUIRE_NOTHROW(realm->refresh()); - REQUIRE(realm->schema() == schema); + auto schema_diff = schema.compare(realm->schema()); + REQUIRE(schema_diff.size() == 1); + REQUIRE(schema_diff == vec{(AddProperty{&*schema.find("object"), + &realm->schema().find("object")->persisted_properties[2]})}); REQUIRE(realm->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); REQUIRE(realm->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); + REQUIRE(realm->schema().find("object")->persisted_properties[2].column_key == col_keys[2]); // Gets the schema from the RealmCoordinator auto realm3 = Realm::get_shared_realm(config); @@ -2410,49 +2645,17 @@ TEST_CASE("migration: Additive") { realm2.reset(); realm3.reset(); + // In case of additive schemas, changes to an external realm are on purpose + // propagated between different realm instances. realm = Realm::get_shared_realm(config); - REQUIRE(realm->schema() == schema); + schema_diff = schema.compare(realm->schema()); + REQUIRE(schema_diff.size() == 1); + REQUIRE(schema_diff == vec{(AddProperty{&*schema.find("object"), + &realm->schema().find("object")->persisted_properties[2]})}); + REQUIRE(realm->schema().find("object")->persisted_properties.size() == 3); REQUIRE(realm->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); REQUIRE(realm->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); - } - - SECTION("obtaining a frozen Realm from before an external schema change") { - auto realm2 = Realm::get_shared_realm(config); - realm->read_group(); - realm2->read_group(); - auto table = ObjectStore::table_for_object_type(realm->read_group(), "object"); - auto col_keys = table->get_column_keys(); - - { - auto write_realm = Realm::get_shared_realm(config); - write_realm->begin_transaction(); - auto table = ObjectStore::table_for_object_type(write_realm->read_group(), "object"); - table->add_column(type_Double, "newcol"); - write_realm->commit_transaction(); - } - - // Before refreshing when we haven't seen the new version at all - auto frozen = realm->freeze(); - REQUIRE(frozen->schema() == schema); - REQUIRE(frozen->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); - REQUIRE(frozen->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); - frozen = Realm::get_frozen_realm(config, realm->read_transaction_version()); - REQUIRE(frozen->schema() == schema); - REQUIRE(frozen->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); - REQUIRE(frozen->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); - - // Refresh the other instance so that the schema cache is updated, and - // then repeat - realm2->refresh(); - - frozen = realm->freeze(); - REQUIRE(frozen->schema() == schema); - REQUIRE(frozen->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); - REQUIRE(frozen->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); - frozen = Realm::get_frozen_realm(config, realm->read_transaction_version()); - REQUIRE(frozen->schema() == schema); - REQUIRE(frozen->schema().find("object")->persisted_properties[0].column_key == col_keys[0]); - REQUIRE(frozen->schema().find("object")->persisted_properties[1].column_key == col_keys[1]); + REQUIRE(realm->schema().find("object")->persisted_properties[2].column_key == col_keys[2]); } SECTION("can have different subsets of columns in different Realm instances") { @@ -2468,11 +2671,11 @@ TEST_CASE("migration: Additive") { auto realm3 = Realm::get_shared_realm(config3); REQUIRE(realm->schema().find("object")->persisted_properties.size() == 2); REQUIRE(realm2->schema().find("object")->persisted_properties.size() == 3); - REQUIRE(realm3->schema().find("object")->persisted_properties.size() == 1); + REQUIRE(realm3->schema().find("object")->persisted_properties.size() == 3); realm->refresh(); realm2->refresh(); - REQUIRE(realm->schema().find("object")->persisted_properties.size() == 2); + REQUIRE(realm->schema().find("object")->persisted_properties.size() == 3); REQUIRE(realm2->schema().find("object")->persisted_properties.size() == 3); // No schema specified; should see all of them @@ -2519,7 +2722,7 @@ TEST_CASE("migration: Additive") { REQUIRE_THROWS_CONTAINING( realm->update_schema(add_property(schema, "object", {"value 3", PropertyType::Float})), "Property 'object.value 3' has been changed from 'int' to 'float'."); - REQUIRE(realm->schema().find("object")->persisted_properties.size() == 2); + REQUIRE(realm->schema().find("object")->persisted_properties.size() == 3); } SECTION("update_schema() does not begin a write transaction when extra columns are present") { diff --git a/test/object-store/object.cpp b/test/object-store/object.cpp index 7197370a329..a5337d311f1 100644 --- a/test/object-store/object.cpp +++ b/test/object-store/object.cpp @@ -164,7 +164,6 @@ TEST_CASE("object") { InMemoryTestFile config; config.cache = false; config.automatic_change_notifications = false; - config.schema_mode = SchemaMode::AdditiveExplicit; config.schema = Schema{ {"table", { @@ -331,7 +330,7 @@ TEST_CASE("object") { advance_and_notify(*r); }; - auto require_change = [&](Object& object, std::optional key_path_array = std::nullopt) { + auto require_change = [&](Object& object, KeyPathArray key_path_array = {}) { auto token = object.add_notification_callback( [&](CollectionChangeSet c) { change = c; @@ -341,7 +340,7 @@ TEST_CASE("object") { return token; }; - auto require_no_change = [&](Object& object, std::optional key_path_array = std::nullopt) { + auto require_no_change = [&](Object& object, KeyPathArray key_path_array = {}) { bool first = true; auto token = object.add_notification_callback( [&](CollectionChangeSet) { @@ -726,79 +725,6 @@ TEST_CASE("object") { } } - SECTION("callback with empty keypatharray") { - SECTION("modifying origin table 'table2', property 'value' " - "while observing related table 'table', property 'value 1' " - "-> does NOT send a notification") { - auto token = require_no_change(object_origin, KeyPathArray()); - - write([&] { - object_origin.set_column_value("value", 105); - }); - } - - SECTION("modifying related table 'table', property 'value 1' " - "while observing related table 'table', property 'value 1' " - "-> does NOT send a notification") { - auto token = require_no_change(object_origin, KeyPathArray()); - - write([&] { - object_target.set_column_value("value 1", 205); - }); - } - - SECTION("modifying related table 'table', property 'value 2' " - "while observing related table 'table', property 'value 1' " - "-> does NOT send a notification") { - auto token = require_no_change(object_origin, KeyPathArray()); - - write([&] { - object_target.set_column_value("value 2", 205); - }); - } - } - - SECTION("callback with empty keypatharray, backlinks") { - SECTION("modifying backlinked table 'table2', property 'value' " - "with empty KeyPathArray " - "-> DOES not send a notification") { - auto token_with_shallow_subscribtion = require_no_change(object_target, KeyPathArray()); - write([&] { - object_origin.set_column_value("value", 105); - }); - } - SECTION("modifying backlinked table 'table2', property 'link' " - "with empty KeyPathArray " - "-> does NOT send a notification") { - auto token_with_empty_key_path_array = require_no_change(object_target, KeyPathArray()); - write([&] { - Obj obj_target2 = table_target->create_object_with_primary_key(300); - Object object_target2(r, obj_target2); - object_origin.set_property_value(d, "link", std::any(object_target2)); - }); - } - SECTION("adding a new origin pointing to the target " - "with empty KeyPathArray " - "-> does NOT send a notification") { - auto token_with_empty_key_path_array = require_no_change(object_target, KeyPathArray()); - write([&] { - Obj obj_origin2 = table_origin->create_object_with_primary_key(300); - Object object_origin2(r, obj_origin2); - object_origin2.set_property_value(d, "link", std::any(object_target)); - }); - } - SECTION("adding a new origin pointing to the target " - "with empty KeyPathArray " - "-> does NOT send a notification") { - auto token_with_empty_key_path_array = require_no_change(object_target, KeyPathArray()); - write([&] { - Obj obj_origin2 = table_origin->create_object_with_primary_key(300); - Object object_origin2(r, obj_origin2); - object_origin2.set_property_value(d, "link", std::any(object_target)); - }); - } - } - SECTION("callbacks on objects with link depth > 4") { r->begin_transaction(); @@ -1775,24 +1701,6 @@ TEST_CASE("object") { REQUIRE(Results(r, r->read_group().get_table("class_nullable object id pk")).size() == 2); } - SECTION("create only requires properties explicitly in the schema") { - config.schema = Schema{{"all types", {{"_id", PropertyType::Int, Property::IsPrimary{true}}}}}; - auto subset_realm = Realm::get_shared_realm(config); - subset_realm->begin_transaction(); - REQUIRE_NOTHROW(Object::create(d, subset_realm, "all types", std::any(AnyDict{{"_id", INT64_C(123)}}))); - subset_realm->commit_transaction(); - - r->refresh(); - auto obj = *r->read_group().get_table("class_all types")->begin(); - REQUIRE(obj.get("_id") == 123); - - // Other columns should have the default unset values - REQUIRE(obj.get("bool") == false); - REQUIRE(obj.get("int") == 0); - REQUIRE(obj.get("float") == 0); - REQUIRE(obj.get("string") == ""); - } - SECTION("getters and setters") { r->begin_transaction(); diff --git a/test/object-store/primitive_list.cpp b/test/object-store/primitive_list.cpp index 8c8c86c3daa..7aead477b0e 100644 --- a/test/object-store/primitive_list.cpp +++ b/test/object-store/primitive_list.cpp @@ -618,16 +618,15 @@ TEMPLATE_TEST_CASE("primitive list", "[primitives]", cf::MixedVal, cf::Int, cf:: } SECTION("remove value from list") { - size_t index = TestType::can_minmax() ? list.find_any(TestType::min()) : 1; advance_and_notify(*r); r->begin_transaction(); - list.remove(index); + list.remove(1); r->commit_transaction(); advance_and_notify(*r); - REQUIRE_INDICES(change.deletions, index); - REQUIRE_INDICES(rchange.deletions, index); - // we removed min(), so it's index 0 for non-optional and 1 for + REQUIRE_INDICES(change.deletions, 1); + REQUIRE_INDICES(rchange.deletions, 1); + // values[1] is min(), so it's index 0 for non-optional and 1 for // optional (as nulls sort to the front) REQUIRE_INDICES(srchange.deletions, TestType::is_optional); } diff --git a/test/object-store/realm.cpp b/test/object-store/realm.cpp index 2830fcc4e5b..61886516ce1 100644 --- a/test/object-store/realm.cpp +++ b/test/object-store/realm.cpp @@ -19,6 +19,8 @@ #include #include +#include + #include "util/event_loop.hpp" #include "util/test_file.hpp" #include "util/test_utils.hpp" @@ -358,6 +360,35 @@ TEST_CASE("SharedRealm: get_shared_realm()") { REQUIRE(old_realm->schema().size() == 1); } + SECTION("should skip schema verification with mode additive and transaction version less than current version") { + + auto realm1 = Realm::get_shared_realm(config); + auto& db1 = TestHelper::get_db(realm1); + auto rt1 = db1->start_read(); + // grab the initial transaction version. + const auto version1 = rt1->get_version_of_current_transaction(); + realm1->close(); + + // update the schema + config.schema_mode = SchemaMode::AdditiveExplicit; + config.schema = Schema{ + {"object", {{"value", PropertyType::Int}}}, + {"object1", {{"value", PropertyType::Int}}}, + }; + auto realm2 = Realm::get_shared_realm(config); + + // no verification if the version chosen is less than the current transaction schema version. + // the schemas should be just merged + TestHelper::begin_read(realm2, version1); + auto& group = realm2->read_group(); + auto schema = realm2->schema(); + REQUIRE(schema == config.schema); + auto table_obj = group.get_table("class_object"); + auto table_obj1 = group.get_table("class_object1"); + REQUIRE(table_obj); // empty schema always has class_object + REQUIRE_FALSE(table_obj1); // class_object1 should not be present + } + SECTION("should sensibly handle opening an uninitialized file without a schema specified") { SECTION("cached") { } @@ -530,20 +561,6 @@ TEST_CASE("SharedRealm: get_shared_realm()") { REQUIRE(object_schema == &*realm->schema().find("object")); } - SECTION("should reuse cached frozen Realm if versions match") { - config.cache = true; - auto realm = Realm::get_shared_realm(config); - realm->read_group(); - auto frozen = realm->freeze(); - frozen->read_group(); - - REQUIRE(frozen != realm); - REQUIRE(realm->read_transaction_version() == frozen->read_transaction_version()); - - REQUIRE(realm->freeze() == frozen); - REQUIRE(Realm::get_frozen_realm(config, realm->read_transaction_version()) == frozen); - } - SECTION("should not use cached frozen Realm if versions don't match") { config.cache = true; auto realm = Realm::get_shared_realm(config); @@ -597,30 +614,6 @@ TEST_CASE("SharedRealm: get_shared_realm()") { REQUIRE(frozen_schema == subset_schema); } - SECTION("frozen realm should have the correct schema even if more properties are added later") { - config.schema_mode = SchemaMode::AdditiveExplicit; - auto full_schema = Schema{ - {"object", {{"value1", PropertyType::Int}, {"value2", PropertyType::Int}}}, - }; - - auto subset_schema = Schema{ - {"object", {{"value1", PropertyType::Int}}}, - }; - - config.schema = subset_schema; - auto realm = Realm::get_shared_realm(config); - realm->read_group(); - - config.schema = full_schema; - auto realm2 = Realm::get_shared_realm(config); - realm2->read_group(); - - auto frozen_realm = realm->freeze(); - REQUIRE(realm->schema() == subset_schema); - REQUIRE(realm2->schema() == full_schema); - REQUIRE(frozen_realm->schema() == subset_schema); - } - SECTION("freeze with orphaned embedded tables") { auto schema = Schema{ {"object1", {{"value", PropertyType::Int}}}, @@ -635,214 +628,6 @@ TEST_CASE("SharedRealm: get_shared_realm()") { } } -TEST_CASE("SharedRealm: schema_subset_mode") { - TestFile config; - config.schema_mode = SchemaMode::AdditiveExplicit; - config.schema_version = 1; - config.schema_subset_mode = SchemaSubsetMode::Complete; - config.encryption_key.clear(); - - // Use a DB directly to simulate changes made by another process - auto db = DB::create(make_in_realm_history(), config.path); - - // Changing the schema version results in update_schema() hitting a very - // different code path for Additive modes, so test both with the schema version - // matching and not matching - auto set_schema_version = GENERATE(false, true); - INFO("Matching schema version: " << set_schema_version); - if (set_schema_version) { - auto tr = db->start_write(); - ObjectStore::set_schema_version(*tr, 1); - tr->commit(); - } - - SECTION("additional properties are added at the end") { - { - auto tr = db->start_write(); - auto table = tr->add_table("class_object"); - for (int i = 0; i < 5; ++i) { - table->add_column(type_Int, util::format("col %1", i)); - } - tr->commit(); - } - - // missing col 0 and 4, and order is different from column order - config.schema = Schema{{"object", - { - {"col 2", PropertyType::Int}, - {"col 3", PropertyType::Int}, - {"col 1", PropertyType::Int}, - }}}; - - auto realm = Realm::get_shared_realm(config); - auto& properties = realm->schema().find("object")->persisted_properties; - REQUIRE(properties.size() == 5); - REQUIRE(properties[0].name == "col 2"); - REQUIRE(properties[1].name == "col 3"); - REQUIRE(properties[2].name == "col 1"); - REQUIRE(properties[3].name == "col 0"); - REQUIRE(properties[4].name == "col 4"); - - for (auto& property : properties) { - REQUIRE(property.column_key != ColKey{}); - } - - config.schema_subset_mode.include_properties = false; - realm = Realm::get_shared_realm(config); - REQUIRE(realm->schema().find("object")->persisted_properties.size() == 3); - } - - SECTION("additional tables are added in sorted order") { - { - auto tr = db->start_write(); - // In reverse order so that just using the table order doesn't - // work accidentally - tr->add_table("class_F")->add_column(type_Int, "value"); - tr->add_table("class_E")->add_column(type_Int, "value"); - tr->add_table("class_D")->add_column(type_Int, "value"); - tr->add_table("class_C")->add_column(type_Int, "value"); - tr->add_table("class_B")->add_column(type_Int, "value"); - tr->add_table("class_A")->add_column(type_Int, "value"); - tr->commit(); - } - - config.schema = Schema{ - {"A", {{"value", PropertyType::Int}}}, - {"E", {{"value", PropertyType::Int}}}, - {"D", {{"value", PropertyType::Int}}}, - }; - auto realm = Realm::get_shared_realm(config); - auto& schema = realm->schema(); - REQUIRE(schema.size() == 6); - REQUIRE(std::is_sorted(schema.begin(), schema.end(), [](auto& a, auto& b) { - return a.name < b.name; - })); - - config.schema_subset_mode.include_types = false; - realm = Realm::get_shared_realm(config); - REQUIRE(realm->schema().size() == 3); - } - - SECTION("schema is updated when refreshing over a schema change") { - config.schema = Schema{{"object", {{"value", PropertyType::Int}}}}; - auto realm = Realm::get_shared_realm(config); - realm->read_group(); - auto& schema = realm->schema(); - - { - auto tr = db->start_write(); - tr->get_table("class_object")->add_column(type_Int, "value 2"); - tr->commit(); - } - - REQUIRE(schema.find("object")->persisted_properties.size() == 1); - realm->refresh(); - REQUIRE(schema.find("object")->persisted_properties.size() == 2); - - { - auto tr = db->start_write(); - tr->add_table("class_object 2")->add_column(type_Int, "value"); - tr->commit(); - } - - REQUIRE(schema.size() == 1); - realm->refresh(); - REQUIRE(schema.size() == 2); - } - - SECTION("schema is updated when schema is modified while not in a read transaction") { - config.schema = Schema{{"object", {{"value", PropertyType::Int}}}}; - auto realm = Realm::get_shared_realm(config); - auto& schema = realm->schema(); - - { - auto tr = db->start_write(); - tr->get_table("class_object")->add_column(type_Int, "value 2"); - tr->commit(); - } - - REQUIRE(schema.find("object")->persisted_properties.size() == 1); - realm->read_group(); - REQUIRE(schema.find("object")->persisted_properties.size() == 2); - realm->invalidate(); - - { - auto tr = db->start_write(); - tr->add_table("class_object 2")->add_column(type_Int, "value"); - tr->commit(); - } - - REQUIRE(schema.size() == 1); - realm->read_group(); - REQUIRE(schema.size() == 2); - } - - SECTION("frozen Realm sees the correct schema for each version") { - config.schema = Schema{{"object", {{"value", PropertyType::Int}}}}; - std::vector> realms; - for (int i = 0; i < 10; ++i) { - realms.push_back(Realm::get_shared_realm(config)); - realms.back()->read_group(); - auto tr = db->start_write(); - tr->add_table(util::format("class_object %1", i))->add_column(type_Int, "value"); - tr->commit(); - } - - auto reset_schema = GENERATE(false, true); - if (reset_schema) { - config.schema.reset(); - } - - for (size_t i = 0; i < 10; ++i) { - auto& r = *realms[i]; - REQUIRE(r.schema().size() == i + 1); - auto frozen = r.freeze(); - REQUIRE(frozen->schema().size() == i + 1); - REQUIRE(frozen->schema_version() == config.schema_version); - frozen = Realm::get_frozen_realm(config, r.read_transaction_version()); - REQUIRE(frozen->schema().size() == i + 1); - REQUIRE(frozen->schema_version() == config.schema_version); - } - - SECTION("schema not set in config") { - config.schema = std::nullopt; - for (size_t i = 0; i < 10; ++i) { - auto& r = *realms[i]; - REQUIRE(r.schema().size() == i + 1); - REQUIRE(r.freeze()->schema().size() == i + 1); - REQUIRE(Realm::get_frozen_realm(config, r.read_transaction_version())->schema().size() == i + 1); - } - } - } - - SECTION("obtaining a frozen realm with an incompatible schema throws") { - config.schema = Schema{{"object", {{"value", PropertyType::Int}}}}; - auto old_realm = Realm::get_shared_realm(config); - old_realm->read_group(); - - { - auto tr = db->start_write(); - tr->add_table("class_object 2")->add_column(type_Int, "value"); - tr->commit(); - } - - config.schema = Schema{ - {"object", {{"value", PropertyType::Int}}}, - {"object 2", {{"value", PropertyType::Int}}}, - }; - auto new_realm = Realm::get_shared_realm(config); - new_realm->read_group(); - - REQUIRE(old_realm->freeze()->schema().size() == 1); - REQUIRE(new_realm->freeze()->schema().size() == 2); - REQUIRE(Realm::get_frozen_realm(config, new_realm->read_transaction_version())->schema().size() == 2); - // Fails because the requested version doesn't have the "object 2" table - // required by the config - REQUIRE_THROWS_AS(Realm::get_frozen_realm(config, old_realm->read_transaction_version()), - InvalidExternalSchemaChangeException); - } -} - #if REALM_ENABLE_SYNC TEST_CASE("Get Realm using Async Open", "[asyncOpen]") { if (!util::EventLoop::has_implementation()) @@ -3440,31 +3225,61 @@ TEST_CASE("SharedRealm: compact on launch") { } struct ModeAutomatic { - static constexpr SchemaMode mode = SchemaMode::Automatic; - static constexpr bool should_call_init_on_version_bump = false; + static SchemaMode mode() + { + return SchemaMode::Automatic; + } + static bool should_call_init_on_version_bump() + { + return false; + } }; struct ModeAdditive { - static constexpr SchemaMode mode = SchemaMode::AdditiveExplicit; - static constexpr bool should_call_init_on_version_bump = false; + static SchemaMode mode() + { + return SchemaMode::AdditiveExplicit; + } + static bool should_call_init_on_version_bump() + { + return false; + } }; struct ModeManual { - static constexpr SchemaMode mode = SchemaMode::Manual; - static constexpr bool should_call_init_on_version_bump = false; + static SchemaMode mode() + { + return SchemaMode::Manual; + } + static bool should_call_init_on_version_bump() + { + return false; + } }; struct ModeSoftResetFile { - static constexpr SchemaMode mode = SchemaMode::SoftResetFile; - static constexpr bool should_call_init_on_version_bump = true; + static SchemaMode mode() + { + return SchemaMode::SoftResetFile; + } + static bool should_call_init_on_version_bump() + { + return true; + } }; struct ModeHardResetFile { - static constexpr SchemaMode mode = SchemaMode::HardResetFile; - static constexpr bool should_call_init_on_version_bump = true; + static SchemaMode mode() + { + return SchemaMode::HardResetFile; + } + static bool should_call_init_on_version_bump() + { + return true; + } }; TEMPLATE_TEST_CASE("SharedRealm: update_schema with initialization_function", "[init][update_schema]", ModeAutomatic, ModeAdditive, ModeManual, ModeSoftResetFile, ModeHardResetFile) { TestFile config; - config.schema_mode = TestType::mode; + config.schema_mode = TestType::mode(); bool initialization_function_called = false; uint64_t schema_version_in_callback = -1; Schema schema_in_callback; @@ -3509,8 +3324,8 @@ TEMPLATE_TEST_CASE("SharedRealm: update_schema with initialization_function", "[ config.schema_version = 1; config.initialization_function = initialization_function; Realm::get_shared_realm(config); - REQUIRE(initialization_function_called == TestType::should_call_init_on_version_bump); - if (TestType::should_call_init_on_version_bump) { + REQUIRE(initialization_function_called == TestType::should_call_init_on_version_bump()); + if (TestType::should_call_init_on_version_bump()) { REQUIRE(schema_version_in_callback == 1); REQUIRE(schema_in_callback.compare(schema).size() == 0); } @@ -3798,97 +3613,6 @@ TEST_CASE("RealmCoordinator: get_unbound_realm()") { } } -TEST_CASE("Immutable Realms") { - TestFile config; // can't be in-memory because we have to write a file to open in immutable mode - config.schema_version = 1; - config.schema = Schema{{"object", {{"value", PropertyType::Int}}}}; - - { - auto realm = Realm::get_shared_realm(config); - realm->begin_transaction(); - realm->read_group().get_table("class_object")->create_object(); - realm->commit_transaction(); - } - - config.schema_mode = SchemaMode::Immutable; - auto realm = Realm::get_shared_realm(config); - realm->read_group(); - - SECTION("unsupported functions") { - SECTION("update_schema()") { - REQUIRE_THROWS_AS(realm->compact(), std::logic_error); - } - SECTION("begin_transaction()") { - REQUIRE_THROWS_AS(realm->begin_transaction(), std::logic_error); - } - SECTION("async_begin_transaction()") { - REQUIRE_THROWS_AS(realm->async_begin_transaction(nullptr), std::logic_error); - } - SECTION("refresh()") { - REQUIRE_THROWS_AS(realm->refresh(), std::logic_error); - } - SECTION("compact()") { - REQUIRE_THROWS_AS(realm->compact(), std::logic_error); - } - } - - SECTION("supported functions") { - SECTION("is_in_transaction()") { - REQUIRE_FALSE(realm->is_in_transaction()); - } - SECTION("is_in_async_transaction()") { - REQUIRE_FALSE(realm->is_in_transaction()); - } - SECTION("freeze()") { - std::shared_ptr frozen; - REQUIRE_NOTHROW(frozen = realm->freeze()); - REQUIRE(frozen->read_group().get_table("class_object")->size() == 1); - REQUIRE_NOTHROW(frozen = Realm::get_frozen_realm(config, realm->read_transaction_version())); - REQUIRE(frozen->read_group().get_table("class_object")->size() == 1); - } - SECTION("notify()") { - REQUIRE_NOTHROW(realm->notify()); - } - SECTION("is_in_read_transaction()") { - REQUIRE(realm->is_in_read_transaction()); - } - SECTION("last_seen_transaction_version()") { - REQUIRE(realm->last_seen_transaction_version() == 1); - } - SECTION("get_number_of_versions()") { - REQUIRE(realm->get_number_of_versions() == 1); - } - SECTION("read_transaction_version()") { - REQUIRE(realm->read_transaction_version() == VersionID{1, 0}); - } - SECTION("current_transaction_version()") { - REQUIRE(realm->current_transaction_version() == VersionID{1, 0}); - } - SECTION("latest_snapshot_version()") { - REQUIRE(realm->latest_snapshot_version() == 1); - } - SECTION("duplicate()") { - auto duplicate = realm->duplicate(); - REQUIRE(duplicate->get_table("class_object")->size() == 1); - } - SECTION("invalidate()") { - REQUIRE_NOTHROW(realm->invalidate()); - REQUIRE_FALSE(realm->is_in_read_transaction()); - REQUIRE(realm->read_group().get_table("class_object")->size() == 1); - } - SECTION("close()") { - REQUIRE_NOTHROW(realm->close()); - REQUIRE(realm->is_closed()); - } - SECTION("has_pending_async_work()") { - REQUIRE_FALSE(realm->has_pending_async_work()); - } - SECTION("wait_for_change()") { - REQUIRE_FALSE(realm->wait_for_change()); - } - } -} - TEST_CASE("KeyPathMapping generation") { TestFile config; realm::query_parser::KeyPathMapping mapping; diff --git a/test/object-store/results.cpp b/test/object-store/results.cpp index ab154cf89bc..27be9d1054d 100644 --- a/test/object-store/results.cpp +++ b/test/object-store/results.cpp @@ -2714,60 +2714,6 @@ TEST_CASE("notifications: results") { } } } - SECTION("callback with empty keypatharray") { - CollectionChangeSet collection_change_set_with_empty_filter; - int notification_calls_with_empty_filter = 0; - auto token_with_empty_filter = results_for_notification_filter.add_notification_callback( - [&](CollectionChangeSet collection_change_set) { - collection_change_set_with_empty_filter = collection_change_set; - ++notification_calls_with_empty_filter; - }, - KeyPathArray()); - advance_and_notify(*r); - REQUIRE(notification_calls_with_empty_filter == 1); - REQUIRE(collection_change_set_with_empty_filter.empty()); - - SECTION("modifying root table 'object', property 'value' " - "-> does NOT send a notification") { - write([&] { - table->get_object(object_keys[0]).set(col_value, 3); - }); - - REQUIRE(notification_calls_with_empty_filter == 1); - REQUIRE(collection_change_set_with_empty_filter.empty()); - } - - SECTION("modifying root table 'object', property 'link' " - "-> does NOT send a notification") { - write([&] { - table->get_object(object_keys[0]).set(col_link, linked_to_table->create_object().get_key()); - }); - REQUIRE(notification_calls_with_empty_filter == 1); - REQUIRE(collection_change_set_with_empty_filter.empty()); - } - - SECTION("inserting 'object' " - "-> DOES send a notification") { - write([&] { - table->create_object(other_table_obj_key).set_all(1); - }); - - REQUIRE(notification_calls_with_empty_filter == 2); - REQUIRE_FALSE(collection_change_set_with_empty_filter.empty()); - REQUIRE_INDICES(collection_change_set_with_empty_filter.insertions, 0); - } - - SECTION("deleting 'object' " - "-> DOES send a notification") { - write([&] { - table->remove_object(object_keys[0]); - }); - - REQUIRE(notification_calls_with_empty_filter == 2); - REQUIRE_FALSE(collection_change_set_with_empty_filter.empty()); - REQUIRE_INDICES(collection_change_set_with_empty_filter.deletions, 0); - } - } SECTION("keypath filter with a backlink") { auto col_second_link = table->get_column_key("second link"); diff --git a/test/object-store/set.cpp b/test/object-store/set.cpp index 217c22e55ed..717947164c0 100644 --- a/test/object-store/set.cpp +++ b/test/object-store/set.cpp @@ -969,76 +969,6 @@ TEMPLATE_TEST_CASE("set", "[set]", CreateNewSet, ReuseSet) }); } } - - SECTION("callback with empty keypatharray") { - auto shallow_require_change = [&] { - auto token = link_set.add_notification_callback( - [&](CollectionChangeSet c) { - change = c; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - auto shallow_require_no_change = [&] { - bool first = true; - auto token = link_set.add_notification_callback( - [&first](CollectionChangeSet) mutable { - REQUIRE(first); - first = false; - }, - KeyPathArray()); - advance_and_notify(*r); - return token; - }; - - SECTION("modifying table 'target', property 'value' " - "-> does NOT send a notification for 'value'") { - auto token2 = shallow_require_no_change(); - write([&] { - target.set(col_table2_value, 23); - }); - } - - SECTION("modifying table 'target', property 'value' " - "-> does NOT send a notification for 'value2'") { - auto token2 = shallow_require_no_change(); - write([&] { - target.set(col_table2_value, 23); - }); - } - - SECTION("modifying the set sends change notifications, shallow") { - Obj target1, target2, target3; - write([&] { - link_set.remove_all(); - }); - REQUIRE(link_set.size() == 0); - write([&]() { - target1 = table2->create_object_with_primary_key(123); - target2 = table2->create_object_with_primary_key(456); - target3 = table2->create_object_with_primary_key(789); - }); - - auto token = shallow_require_change(); - - write([&]() { - CHECK(link_set.insert(target1).second); - CHECK(!link_set.insert(target1).second); - CHECK(link_set.insert(target2).second); - CHECK(link_set.insert(target3).second); - }); - - REQUIRE(link_set.size() == 3); - - write([&] { - CHECK(link_set.remove(target2).second); - }); - REQUIRE_INDICES(change.deletions, 1); - REQUIRE(!change.collection_was_cleared); - } - } } } diff --git a/test/object-store/sync/app.cpp b/test/object-store/sync/app.cpp index 6a69d1e71d3..547f90f179b 100644 --- a/test/object-store/sync/app.cpp +++ b/test/object-store/sync/app.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include "collection_fixtures.hpp" #include "sync_test_utils.hpp" @@ -2009,7 +2008,7 @@ constexpr size_t minus_25_percent(size_t val) } TEST_CASE("app: sync integration", "[sync][app]") { - auto logger = std::make_shared(realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL); + auto logger = std::make_unique(realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL); const auto schema = default_app_config("").schema; @@ -2125,37 +2124,11 @@ TEST_CASE("app: sync integration", "[sync][app]") { }); } // Optional handler for the request and response before it is returned to completion - std::function response_hook; + util::UniqueFunction response_hook; // Optional handler for the request before it is sent to the server - std::function request_hook; + util::UniqueFunction request_hook; // Optional Response object to return immediately instead of communicating with the server - std::optional simulated_response; - }; - - struct HookedSocketProvider : public sync::websocket::DefaultSocketProvider { - HookedSocketProvider(const std::shared_ptr& logger, const std::string user_agent, - AutoStart auto_start = AutoStart{true}) - : DefaultSocketProvider(logger, user_agent, auto_start) - { - } - - std::unique_ptr connect(std::unique_ptr observer, - sync::WebSocketEndpoint&& endpoint) override - { - int status_code = 101; - std::string body; - bool use_simulated_response = websocket_connect_func && websocket_connect_func(status_code, body); - - auto websocket = DefaultSocketProvider::connect(std::move(observer), std::move(endpoint)); - if (use_simulated_response) { - auto default_websocket = static_cast(websocket.get()); - if (default_websocket) - default_websocket->force_handshake_response_for_testing(status_code, body); - } - return websocket; - } - - std::function websocket_connect_func; + util::Optional simulated_response; }; { @@ -2224,17 +2197,10 @@ TEST_CASE("app: sync integration", "[sync][app]") { if (request.url.find("https://") != std::string::npos) { redirect_scheme = "https://"; } - // using local baas if (request.url.find("127.0.0.1:9090") != std::string::npos) { redirect_host = "localhost:9090"; original_host = "127.0.0.1:9090"; } - // using baas docker - can't test redirect - else if (request.url.find("mongodb-realm:9090") != std::string::npos) { - redirect_host = "mongodb-realm:9090"; - original_host = "mongodb-realm:9090"; - } - redirect_url = redirect_scheme + redirect_host; logger->trace("redirect_url (%1): %2", request_count, redirect_url); request_count++; @@ -2270,7 +2236,7 @@ TEST_CASE("app: sync integration", "[sync][app]") { logger->trace("request.url (%1): %2", request_count, request.url); REQUIRE(request.url.find(redirect_scheme + original_host) != std::string::npos); // Let the init_app_metadata request go through - redir_transport->simulated_response.reset(); + redir_transport->simulated_response = util::none; request_count++; } else if (request_count == 5) { @@ -2286,7 +2252,7 @@ TEST_CASE("app: sync integration", "[sync][app]") { logger->trace("WS Hostname: %1", app_metadata->ws_hostname); REQUIRE(app_metadata->hostname.find(original_host) != std::string::npos); REQUIRE(request.url.find(redirect_scheme + original_host) != std::string::npos); - redir_transport->simulated_response.reset(); + redir_transport->simulated_response = util::none; // Validate the retry count tracked in the original message REQUIRE(request.redirect_count == 3); request_count++; @@ -2344,258 +2310,6 @@ TEST_CASE("app: sync integration", "[sync][app]") { }); } } - SECTION("Test app redirect with no metadata") { - std::unique_ptr app_session; - std::string base_file_path = util::make_temp_dir() + random_string(10); - auto redir_transport = std::make_shared(); - AutoVerifiedEmailCredentials creds, creds2; - - auto app_config = get_config(redir_transport, session.app_session()); - set_app_config_defaults(app_config, redir_transport); - - util::try_make_dir(base_file_path); - SyncClientConfig sc_config; - sc_config.base_file_path = base_file_path; - sc_config.log_level = realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL; - sc_config.metadata_mode = realm::SyncManager::MetadataMode::NoMetadata; - - // initialize app and sync client - auto redir_app = app::App::get_uncached_app(app_config, sc_config); - - int request_count = 0; - // redirect URL is localhost or 127.0.0.1 depending on what the initial value is - std::string original_host = "localhost:9090"; - std::string original_scheme = "http://"; - std::string websocket_url = "ws://some-websocket:9090"; - std::string original_url; - redir_transport->request_hook = [&](const Request& request) { - if (request_count == 0) { - logger->trace("request.url (%1): %2", request_count, request.url); - if (request.url.find("https://") != std::string::npos) { - original_scheme = "https://"; - } - // using local baas - if (request.url.find("127.0.0.1:9090") != std::string::npos) { - original_host = "127.0.0.1:9090"; - } - // using baas docker - else if (request.url.find("mongodb-realm:9090") != std::string::npos) { - original_host = "mongodb-realm:9090"; - } - original_url = original_scheme + original_host; - logger->trace("original_url (%1): %2", request_count, original_url); - } - else if (request_count == 1) { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(!request.redirect_count); - redir_transport->simulated_response = { - 308, - 0, - {{"Location", "http://somehost:9090"}, {"Content-Type", "application/json"}}, - "Some body data"}; - } - else if (request_count == 2) { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(request.url.find("http://somehost:9090") != std::string::npos); - REQUIRE(request.url.find("location") != std::string::npos); - // app hostname will be updated via the metadata info - redir_transport->simulated_response = { - static_cast(sync::HTTPStatus::Ok), - 0, - {{"Content-Type", "application/json"}}, - util::format("{\"deployment_model\":\"GLOBAL\",\"location\":\"US-VA\",\"hostname\":\"%1\",\"ws_" - "hostname\":\"%2\"}", - original_url, websocket_url)}; - } - else { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(request.url.find(original_url) != std::string::npos); - redir_transport->simulated_response.reset(); - } - request_count++; - }; - - // This will be successful after a couple of retries due to the redirect response - redir_app->provider_client().register_email( - creds.email, creds.password, [&](util::Optional error) { - REQUIRE(!error); - }); - REQUIRE(!redir_app->sync_manager()->app_metadata()); // no stored app metadata - REQUIRE(redir_app->sync_manager()->sync_route().find(websocket_url) != std::string::npos); - - // Register another email address and verify location data isn't requested again - request_count = 0; - redir_transport->request_hook = [&](const Request& request) { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response.reset(); - REQUIRE(request.url.find("location") == std::string::npos); - request_count++; - }; - - redir_app->provider_client().register_email( - creds2.email, creds2.password, [&](util::Optional error) { - REQUIRE(!error); - }); - } - - SECTION("Test websocket redirect with existing session") { - std::string original_host = "localhost:9090"; - std::string redirect_scheme = "http://"; - std::string websocket_scheme = "ws://"; - std::string redirect_host = "127.0.0.1:9090"; - std::string redirect_url = "http://127.0.0.1:9090"; - - auto redir_transport = std::make_shared(); - auto redir_provider = std::make_shared(logger, ""); - - // Use the transport to grab the current url so it can be converted - redir_transport->request_hook = [&](const Request& request) { - if (request.url.find("https://") != std::string::npos) { - redirect_scheme = "https://"; - websocket_scheme = "wss://"; - } - // using local baas - if (request.url.find("127.0.0.1:9090") != std::string::npos) { - redirect_host = "localhost:9090"; - original_host = "127.0.0.1:9090"; - } - // using baas docker - can't test redirect - else if (request.url.find("mongodb-realm:9090") != std::string::npos) { - redirect_host = "mongodb-realm:9090"; - original_host = "mongodb-realm:9090"; - } - - redirect_url = redirect_scheme + redirect_host; - logger->trace("redirect_url: %1", redirect_url); - }; - - auto base_url = get_base_url(); - auto server_app_config = minimal_app_config(base_url, "websocket_redirect", schema); - TestAppSession test_session(create_app(server_app_config), redir_transport, DeleteApp{true}, - realm::ReconnectMode::normal, redir_provider); - auto partition = random_string(100); - auto user1 = test_session.app()->current_user(); - SyncTestFile r_config(user1, partition, schema); - // Overrride the default - r_config.sync_config->error_handler = [](std::shared_ptr, SyncError error) { - if (error.error_code == sync::make_error_code(realm::sync::ProtocolError::bad_authentication)) { - util::format(std::cerr, "Websocket redirect test: User logged out\n"); - return; - } - util::format(std::cerr, "An unexpected sync error was caught by the default SyncTestFile handler: '%1'\n", - error.message); - abort(); - }; - - auto r = Realm::get_shared_realm(r_config); - - REQUIRE(!wait_for_download(*r)); - - SECTION("Valid websocket redirect") { - auto sync_manager = test_session.app()->sync_manager(); - auto sync_session = sync_manager->get_existing_session(r->config().path); - sync_session->pause(); - - int connect_count = 0; - redir_provider->websocket_connect_func = [&connect_count](int& status_code, std::string& body) { - if (connect_count++ > 0) - return false; - - status_code = static_cast(sync::HTTPStatus::PermanentRedirect); - body = ""; - return true; - }; - int request_count = 0; - redir_transport->request_hook = [&](const Request& request) { - if (request_count++ == 0) { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(!request.redirect_count); - redir_transport->simulated_response = { - static_cast(sync::HTTPStatus::PermanentRedirect), - 0, - {{"Location", redirect_url}, {"Content-Type", "application/json"}}, - "Some body data"}; - } - else if (request.url.find("location") != std::string::npos) { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response = { - static_cast(sync::HTTPStatus::Ok), - 0, - {{"Content-Type", "application/json"}}, - util::format( - "{\"deployment_model\":\"GLOBAL\",\"location\":\"US-VA\",\"hostname\":\"%2%1\",\"ws_" - "hostname\":\"%3%1\"}", - redirect_host, redirect_scheme, websocket_scheme)}; - } - else { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response.reset(); - } - }; - - sync_session->resume(); - REQUIRE(!wait_for_download(*r)); - - // Verify session is using the updated server url from the redirect - auto server_url = sync_session->full_realm_url(); - logger->trace("FULL_REALM_URL: %1", server_url); - REQUIRE((server_url && server_url->find(redirect_host) != std::string::npos)); - } - SECTION("Websocket redirect logs out user") { - auto sync_manager = test_session.app()->sync_manager(); - auto sync_session = sync_manager->get_existing_session(r->config().path); - sync_session->pause(); - - int connect_count = 0; - redir_provider->websocket_connect_func = [&connect_count](int& status_code, std::string& body) { - if (connect_count++ > 0) - return false; - - status_code = static_cast(sync::HTTPStatus::MovedPermanently); - body = ""; - return true; - }; - int request_count = 0; - redir_transport->request_hook = [&](const Request& request) { - if (request_count++ == 0) { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(!request.redirect_count); - redir_transport->simulated_response = { - static_cast(sync::HTTPStatus::MovedPermanently), - 0, - {{"Location", redirect_url}, {"Content-Type", "application/json"}}, - "Some body data"}; - } - else if (request.url.find("location") != std::string::npos) { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response = { - static_cast(sync::HTTPStatus::Ok), - 0, - {{"Content-Type", "application/json"}}, - util::format( - "{\"deployment_model\":\"GLOBAL\",\"location\":\"US-VA\",\"hostname\":\"%2%1\",\"ws_" - "hostname\":\"%3%1\"}", - redirect_host, redirect_scheme, websocket_scheme)}; - } - else if (request.url.find("auth/session") != std::string::npos) { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response = {static_cast(sync::HTTPStatus::Unauthorized), - 0, - {{"Content-Type", "application/json"}}, - ""}; - } - else { - logger->trace("request.url (%1): %2", request_count, request.url); - redir_transport->simulated_response.reset(); - } - }; - - sync_session->resume(); - REQUIRE(wait_for_download(*r)); - REQUIRE(!user1->is_logged_in()); - } - } - SECTION("Fast clock on client") { { SyncTestFile config(app, partition, schema); @@ -2976,7 +2690,7 @@ TEST_CASE("app: sync integration", "[sync][app]") { std::mutex mutex; bool done = false; auto r = Realm::get_shared_realm(config); - r->sync_session()->pause(); + r->sync_session()->close(); // Create 26 MB worth of dogs in 26 transactions, which should work but // will result in an error from the server if the changesets are batched @@ -2996,7 +2710,7 @@ TEST_CASE("app: sync integration", "[sync][app]") { REQUIRE(!ec); done = true; }); - r->sync_session()->resume(); + r->sync_session()->revive_if_needed(); // If we haven't gotten an error in more than 5 minutes, then something has gone wrong // and we should fail the test. @@ -3039,33 +2753,6 @@ TEST_CASE("app: sync integration", "[sync][app]") { REQUIRE(error.server_requests_action == sync::ProtocolErrorInfo::Action::ClientReset); } - SECTION("freezing realm does not resume session") { - SyncTestFile config(app, partition, schema); - auto realm = Realm::get_shared_realm(config); - wait_for_download(*realm); - - auto state = realm->sync_session()->state(); - REQUIRE(state == SyncSession::State::Active); - - realm->sync_session()->pause(); - state = realm->sync_session()->state(); - REQUIRE(state == SyncSession::State::Paused); - - realm->read_group(); - - { - auto frozen = realm->freeze(); - REQUIRE(realm->sync_session() == realm->sync_session()); - REQUIRE(realm->sync_session()->state() == SyncSession::State::Paused); - } - - { - auto frozen = Realm::get_frozen_realm(config, realm->read_transaction_version()); - REQUIRE(realm->sync_session() == realm->sync_session()); - REQUIRE(realm->sync_session()->state() == SyncSession::State::Paused); - } - } - SECTION("validation") { SyncTestFile config(app, partition, schema); diff --git a/test/object-store/sync/client_reset.cpp b/test/object-store/sync/client_reset.cpp index f2acb859e82..3173cc840d4 100644 --- a/test/object-store/sync/client_reset.cpp +++ b/test/object-store/sync/client_reset.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include @@ -109,158 +108,6 @@ TableRef get_table(Realm& realm, StringData object_type) namespace cf = realm::collection_fixtures; using reset_utils::create_object; -TEST_CASE("sync: large reset with recovery is restartable", "[client reset]") { - const reset_utils::Partition partition{"realm_id", random_string(20)}; - Property partition_prop = {partition.property_name, PropertyType::String | PropertyType::Nullable}; - Schema schema{ - {"object", - { - {"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"value", PropertyType::String}, - partition_prop, - }}, - }; - - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); - server_app_config.partition_key = partition_prop; - TestAppSession test_app_session(create_app(server_app_config)); - auto app = test_app_session.app(); - - create_user_and_log_in(app); - SyncTestFile realm_config(app->current_user(), partition.value, schema); - realm_config.sync_config->client_resync_mode = ClientResyncMode::Recover; - realm_config.sync_config->error_handler = [&](std::shared_ptr, SyncError err) { - if (err.error_code == util::make_error_code(util::MiscExtErrors::end_of_input)) { - return; - } - - if (err.server_requests_action == sync::ProtocolErrorInfo::Action::Warning || - err.server_requests_action == sync::ProtocolErrorInfo::Action::Transient) { - return; - } - - FAIL(util::format("got error from server: %1: %2", err.error_code.value(), err.message)); - }; - - auto realm = Realm::get_shared_realm(realm_config); - std::vector expected_obj_ids; - { - auto obj_id = ObjectId::gen(); - expected_obj_ids.push_back(obj_id); - realm->begin_transaction(); - CppContext c(realm); - Object::create(c, realm, "object", - std::any(AnyDict{{"_id", obj_id}, - {"value", std::string{"hello world"}}, - {partition.property_name, partition.value}})); - realm->commit_transaction(); - wait_for_upload(*realm); - reset_utils::wait_for_object_to_persist_to_atlas(app->current_user(), test_app_session.app_session(), - "object", {{"_id", obj_id}}); - realm->sync_session()->pause(); - } - - reset_utils::trigger_client_reset(test_app_session.app_session(), realm); - { - SyncTestFile realm_config(app->current_user(), partition.value, schema); - auto second_realm = Realm::get_shared_realm(realm_config); - - second_realm->begin_transaction(); - CppContext c(second_realm); - for (size_t i = 0; i < 100; ++i) { - auto obj_id = ObjectId::gen(); - expected_obj_ids.push_back(obj_id); - Object::create(c, second_realm, "object", - std::any(AnyDict{{"_id", obj_id}, - {"value", random_string(1024 * 128)}, - {partition.property_name, partition.value}})); - } - second_realm->commit_transaction(); - - wait_for_upload(*second_realm); - } - - realm->sync_session()->resume(); - timed_wait_for([&] { - return util::File::exists(_impl::ClientResetOperation::get_fresh_path_for(realm_config.path)); - }); - realm->sync_session()->pause(); - realm->sync_session()->resume(); - wait_for_upload(*realm); - wait_for_download(*realm); - - realm->refresh(); - auto table = realm->read_group().get_table("class_object"); - REQUIRE(table->size() == expected_obj_ids.size()); - std::vector found_object_ids; - for (const auto& obj : *table) { - found_object_ids.push_back(obj.get_primary_key().get_object_id()); - } - - std::stable_sort(expected_obj_ids.begin(), expected_obj_ids.end()); - std::stable_sort(found_object_ids.begin(), found_object_ids.end()); - REQUIRE(expected_obj_ids == found_object_ids); -} - -TEST_CASE("sync: pending client resets are cleared when downloads are complete", "[client reset]") { - const reset_utils::Partition partition{"realm_id", random_string(20)}; - Property partition_prop = {partition.property_name, PropertyType::String | PropertyType::Nullable}; - Schema schema{ - {"object", - { - {"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"value", PropertyType::Int}, - partition_prop, - }}, - }; - - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); - server_app_config.partition_key = partition_prop; - TestAppSession test_app_session(create_app(server_app_config)); - auto app = test_app_session.app(); - - create_user_and_log_in(app); - SyncTestFile realm_config(app->current_user(), partition.value, schema); - realm_config.sync_config->client_resync_mode = ClientResyncMode::Recover; - realm_config.sync_config->error_handler = [&](std::shared_ptr, SyncError err) { - if (err.error_code == sync::websocket::make_error_code(ErrorCodes::ReadError)) { - return; - } - - if (err.server_requests_action == sync::ProtocolErrorInfo::Action::Warning || - err.server_requests_action == sync::ProtocolErrorInfo::Action::Transient) { - return; - } - - FAIL(util::format("got error from server: %1: %2", err.error_code.value(), err.message)); - }; - - auto realm = Realm::get_shared_realm(realm_config); - auto obj_id = ObjectId::gen(); - { - realm->begin_transaction(); - CppContext c(realm); - Object::create( - c, realm, "object", - std::any(AnyDict{{"_id", obj_id}, {"value", int64_t(5)}, {partition.property_name, partition.value}})); - realm->commit_transaction(); - wait_for_upload(*realm); - } - wait_for_download(*realm, std::chrono::minutes(10)); - - reset_utils::trigger_client_reset(test_app_session.app_session(), realm); - - wait_for_download(*realm, std::chrono::minutes(10)); - - reset_utils::trigger_client_reset(test_app_session.app_session(), realm); - - wait_for_download(*realm, std::chrono::minutes(10)); -} - TEST_CASE("sync: client reset", "[client reset]") { if (!util::EventLoop::has_implementation()) return; @@ -381,8 +228,6 @@ TEST_CASE("sync: client reset", "[client reset]") { REQUIRE(before->is_frozen()); REQUIRE(before->read_group().get_table("class_object")); REQUIRE(before->config().path == local_config.path); - REQUIRE_FALSE(before->schema().empty()); - REQUIRE(before->schema_version() != ObjectStore::NotVersioned); REQUIRE(util::File::exists(local_config.path)); }; local_config.sync_config->notify_after_client_reset = [&](SharedRealm before, ThreadSafeReference after_ref, @@ -869,10 +714,8 @@ TEST_CASE("sync: client reset", "[client reset]") { temp_config.persist(); temp_config.sync_config->client_resync_mode = ClientResyncMode::DiscardLocal; config_copy = std::make_unique(*temp_config.sync_config); - config_copy->notify_before_client_reset = [&](SharedRealm before_realm) { + config_copy->notify_before_client_reset = [&](SharedRealm) { std::lock_guard lock(mtx); - REQUIRE(before_realm); - REQUIRE(before_realm->schema_version() != ObjectStore::NotVersioned); ++before_callback_invoctions_2; }; config_copy->notify_after_client_reset = [&](SharedRealm, ThreadSafeReference, bool) { @@ -880,13 +723,11 @@ TEST_CASE("sync: client reset", "[client reset]") { ++after_callback_invocations_2; }; - temp_config.sync_config->notify_before_client_reset = [&](SharedRealm before_realm) { + temp_config.sync_config->notify_before_client_reset = [&](SharedRealm) { std::lock_guard lock(mtx); ++before_callback_invoctions; REQUIRE(session); REQUIRE(config_copy); - REQUIRE(before_realm); - REQUIRE(before_realm->schema_version() != ObjectStore::NotVersioned); session->update_configuration(*config_copy); }; @@ -1747,98 +1588,6 @@ TEST_CASE("sync: client reset", "[client reset]") { } // end: The server can prohibit recovery } -TEST_CASE("sync: Client reset during async open", "[client reset]") { - const reset_utils::Partition partition{"realm_id", random_string(20)}; - Property partition_prop = {partition.property_name, PropertyType::String | PropertyType::Nullable}; - Schema schema{ - {"object", - { - {"_id", PropertyType::ObjectId, Property::IsPrimary{true}}, - {"value", PropertyType::String}, - partition_prop, - }}, - }; - - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto server_app_config = minimal_app_config(base_url, "client_reset_tests", schema); - server_app_config.partition_key = partition_prop; - TestAppSession test_app_session(create_app(server_app_config)); - auto app = test_app_session.app(); - - auto before_callback_called = util::make_promise_future(); - auto after_callback_called = util::make_promise_future(); - create_user_and_log_in(app); - SyncTestFile realm_config(app->current_user(), partition.value, std::nullopt, - [](std::shared_ptr, SyncError) { /*noop*/ }); - realm_config.sync_config->client_resync_mode = ClientResyncMode::Recover; - - realm_config.sync_config->on_sync_client_event_hook = - [&, client_reset_triggered = false](std::weak_ptr weak_sess, - const SyncClientHookData& event_data) mutable { - auto sess = weak_sess.lock(); - if (!sess) { - return SyncClientHookAction::NoAction; - } - if (sess->path() != realm_config.path) { - return SyncClientHookAction::NoAction; - } - - if (event_data.event != SyncClientHookEvent::DownloadMessageReceived) { - return SyncClientHookAction::NoAction; - } - - if (client_reset_triggered) { - return SyncClientHookAction::NoAction; - } - client_reset_triggered = true; - reset_utils::trigger_client_reset(test_app_session.app_session()); - return SyncClientHookAction::EarlyReturn; - }; - - // Expected behaviour is that the frozen realm passed in the callback should have no - // schema initialized if a client reset happens during an async open and the realm has never been opened before. - // SDK's should handle any edge cases which require the use of a schema i.e - // calling set_schema_subset(...) - realm_config.sync_config->notify_before_client_reset = - [promise = util::CopyablePromiseHolder(std::move(before_callback_called.promise))]( - std::shared_ptr realm) mutable { - CHECK(realm->schema_version() == ObjectStore::NotVersioned); - promise.get_promise().emplace_value(); - }; - - realm_config.sync_config->notify_after_client_reset = - [promise = util::CopyablePromiseHolder(std::move(after_callback_called.promise))]( - std::shared_ptr realm, ThreadSafeReference, bool) mutable { - CHECK(realm->schema_version() == ObjectStore::NotVersioned); - promise.get_promise().emplace_value(); - }; - - auto realm_task = Realm::get_synchronized_realm(realm_config); - auto realm_pf = util::make_promise_future(); - realm_task->start([promise_holder = util::CopyablePromiseHolder(std::move(realm_pf.promise))]( - ThreadSafeReference ref, std::exception_ptr ex) mutable { - auto promise = promise_holder.get_promise(); - if (ex) { - try { - std::rethrow_exception(ex); - } - catch (...) { - promise.set_error(exception_to_status()); - } - return; - } - auto realm = Realm::get_shared_realm(std::move(ref)); - if (!realm) { - promise.set_error({ErrorCodes::RuntimeError, "could not get realm from threadsaferef"}); - } - promise.emplace_value(std::move(realm)); - }); - auto realm = realm_pf.future.get(); - before_callback_called.future.get(); - after_callback_called.future.get(); -} - #endif // REALM_ENABLE_AUTH_TESTS namespace cf = realm::collection_fixtures; diff --git a/test/object-store/sync/flx_sync.cpp b/test/object-store/sync/flx_sync.cpp index cecca191b67..b1ef49f4322 100644 --- a/test/object-store/sync/flx_sync.cpp +++ b/test/object-store/sync/flx_sync.cpp @@ -49,18 +49,6 @@ using namespace std::string_literals; -namespace realm { - -class TestHelper { -public: - static DBRef& get_db(SharedRealm const& shared_realm) - { - return Realm::Internal::get_db(*shared_realm); - } -}; - -} // namespace realm - namespace realm::app { namespace { @@ -445,7 +433,6 @@ TEST_CASE("flx: client reset", "[sync][flx][app][client reset]") { {"sum_of_list_field", sum}})); realm->commit_transaction(); - wait_for_upload(*realm); return pk_of_added_object; }) ->make_local_changes([&](SharedRealm local_realm) { @@ -1212,7 +1199,7 @@ TEST_CASE("flx: query on non-queryable field results in query error message", "[ SECTION("Bad query after bad query") { harness->do_with_new_realm([&](SharedRealm realm) { auto sync_session = realm->sync_session(); - sync_session->pause(); + sync_session->close(); auto subs = create_subscription(realm, "class_TopLevel", "non_queryable_field", [](auto q, auto c) { return q.equal(c, "bar"); @@ -1221,7 +1208,7 @@ TEST_CASE("flx: query on non-queryable field results in query error message", "[ return q.equal(c, "bar"); }); - sync_session->resume(); + sync_session->revive_if_needed(); auto sub_res = subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get_no_throw(); auto sub_res2 = @@ -1414,7 +1401,7 @@ TEST_CASE("flx: change-of-query history divergence", "[sync][flx][app]") { wait_for_download(*realm); // Now disconnect the sync session - realm->sync_session()->pause(); + realm->sync_session()->close(); // And move the "foo" object created above into view and create a different diverging copy of it locally. auto mut_subs = realm->get_latest_subscription_set().make_mutable_copy(); @@ -1431,7 +1418,7 @@ TEST_CASE("flx: change-of-query history divergence", "[sync][flx][app]") { realm->commit_transaction(); // Reconnect the sync session and wait for the subscription that moved "foo" into view to be fully synchronized. - realm->sync_session()->resume(); + realm->sync_session()->revive_if_needed(); wait_for_upload(*realm); wait_for_download(*realm); subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get(); @@ -1486,7 +1473,7 @@ TEST_CASE("flx: writes work offline", "[sync][flx][app]") { wait_for_upload(*realm); wait_for_download(*realm); - sync_session->pause(); + sync_session->close(); // Make it so the subscriptions only match the "foo" object { @@ -1524,7 +1511,7 @@ TEST_CASE("flx: writes work offline", "[sync][flx][app]") { realm->commit_transaction(); } - sync_session->resume(); + sync_session->revive_if_needed(); wait_for_upload(*realm); wait_for_download(*realm); @@ -1878,7 +1865,7 @@ TEST_CASE("flx: bootstrap batching prevents orphan documents", "[sync][flx][app] } if (data.query_version == 1 && data.batch_state == sync::DownloadBatchState::MoreToCome) { - session->force_close(); + session->close(); promise->emplace_value(); return SyncClientHookAction::EarlyReturn; } @@ -1957,7 +1944,7 @@ TEST_CASE("flx: bootstrap batching prevents orphan documents", "[sync][flx][app] } if (data.query_version == 1 && data.batch_state == sync::DownloadBatchState::LastInBatch) { - session->force_close(); + session->close(); promise->emplace_value(); return SyncClientHookAction::EarlyReturn; } @@ -2449,7 +2436,7 @@ TEST_CASE("flx: bootstraps contain all changes", "[sync][flx][app]") { REQUIRE(table->find_primary_key(bar_obj_id)); REQUIRE_FALSE(table->find_primary_key(bizz_obj_id)); - sess->pause(); + sess->close(); promise.get_promise().emplace_value(); return SyncClientHookAction::NoAction; }; @@ -2484,7 +2471,7 @@ TEST_CASE("flx: bootstraps contain all changes", "[sync][flx][app]") { sub_set.refresh(); REQUIRE(sub_set.state() == sync::SubscriptionSet::State::AwaitingMark); - problem_realm->sync_session()->resume(); + problem_realm->sync_session()->revive_if_needed(); sub_complete_future.get(); wait_for_advance(*problem_realm); @@ -2860,129 +2847,6 @@ TEST_CASE("flx: compensating write errors get re-sent across sessions", "[sync][ REQUIRE(top_level_table->is_empty()); } -TEST_CASE("flx: bootstrap changesets are applied continuously", "[sync][flx][app]") { - FLXSyncTestHarness harness("flx_bootstrap_batching", {g_large_array_schema, {"queryable_int_field"}}); - fill_large_array_schema(harness); - - std::unique_ptr th; - sync::version_type user_commit_version = UINT_FAST64_MAX; - sync::version_type bootstrap_version = UINT_FAST64_MAX; - SharedRealm realm; - std::condition_variable cv; - std::mutex mutex; - bool allow_to_commit = false; - - SyncTestFile config(harness.app()->current_user(), harness.schema(), SyncConfig::FLXSyncEnabled{}); - auto [interrupted_promise, interrupted] = util::make_promise_future(); - auto shared_promise = std::make_shared>(std::move(interrupted_promise)); - config.sync_config->on_sync_client_event_hook = - [promise = std::move(shared_promise), &th, &realm, &user_commit_version, &bootstrap_version, &cv, &mutex, - &allow_to_commit](std::weak_ptr weak_session, const SyncClientHookData& data) { - if (data.query_version == 0) { - return SyncClientHookAction::NoAction; - } - if (data.event != SyncClientHookEvent::DownloadMessageIntegrated) { - return SyncClientHookAction::NoAction; - } - auto session = weak_session.lock(); - if (!session) { - return SyncClientHookAction::NoAction; - } - if (data.batch_state != sync::DownloadBatchState::MoreToCome) { - // Read version after bootstrap is done. - auto db = TestHelper::get_db(realm); - ReadTransaction rt(db); - bootstrap_version = rt.get_version(); - { - std::lock_guard lock(mutex); - allow_to_commit = true; - } - cv.notify_one(); - session->force_close(); - promise->emplace_value(); - return SyncClientHookAction::NoAction; - } - - if (th) { - return SyncClientHookAction::NoAction; - } - - auto func = [&] { - // Attempt to commit a local change after the first bootstrap batch was committed. - auto db = TestHelper::get_db(realm); - WriteTransaction wt(db); - TableRef table = wt.get_table("class_TopLevel"); - table->create_object_with_primary_key(ObjectId::gen()); - { - std::unique_lock lock(mutex); - // Wait to commit until we read the final bootstrap version. - cv.wait(lock, [&] { - return allow_to_commit; - }); - } - user_commit_version = wt.commit(); - }; - th = std::make_unique(std::move(func)); - - return SyncClientHookAction::NoAction; - }; - - realm = Realm::get_shared_realm(config); - auto table = realm->read_group().get_table("class_TopLevel"); - Query query(table); - { - auto new_subs = realm->get_latest_subscription_set().make_mutable_copy(); - new_subs.insert_or_assign(query); - new_subs.commit(); - } - interrupted.get(); - th->join(); - - // The user commit is the last one. - CHECK(user_commit_version == bootstrap_version + 1); -} - - -TEST_CASE("flx: really big bootstraps", "[sync][flx][app]") { - FLXSyncTestHarness harness("harness"); - - std::vector expected_obj_ids; - harness.load_initial_data([&](SharedRealm realm) { - realm->cancel_transaction(); - for (size_t n = 0; n < 10; ++n) { - realm->begin_transaction(); - for (size_t i = 0; i < 100; ++i) { - expected_obj_ids.push_back(ObjectId::gen()); - auto& obj_id = expected_obj_ids.back(); - CppContext c(realm); - Object::create(c, realm, "TopLevel", - std::any(AnyDict{{"_id", obj_id}, - {"queryable_str_field", "foo"s}, - {"queryable_int_field", static_cast(5)}, - {"non_queryable_field", random_string(1024 * 128)}})); - } - realm->commit_transaction(); - } - realm->begin_transaction(); - }); - - SyncTestFile target(harness.app()->current_user(), harness.schema(), SyncConfig::FLXSyncEnabled{}); - auto error_pf = util::make_promise_future(); - target.sync_config->error_handler = [promise = util::CopyablePromiseHolder(std::move(error_pf.promise))]( - std::shared_ptr, SyncError err) mutable { - promise.get_promise().emplace_value(std::move(err)); - }; - auto realm = Realm::get_shared_realm(target); - auto mut_subs = realm->get_latest_subscription_set().make_mutable_copy(); - mut_subs.insert_or_assign(Query(realm->read_group().get_table("class_TopLevel"))); - auto subs = mut_subs.commit(); - - // TODO when BAAS-19105 is fixed we should be able to just wait for bootstrapping to be complete. For now though, - // check that we get the error code we expect. - auto err = error_pf.future.get(); - REQUIRE(err.error_code == sync::ClientError::bad_changeset_size); -} - } // namespace realm::app #endif // REALM_ENABLE_AUTH_TESTS diff --git a/test/object-store/sync/flx_sync_harness.hpp b/test/object-store/sync/flx_sync_harness.hpp index 9e98afea542..9b8e911bb8a 100644 --- a/test/object-store/sync/flx_sync_harness.hpp +++ b/test/object-store/sync/flx_sync_harness.hpp @@ -72,20 +72,17 @@ class FLXSyncTestHarness { ServerSchema server_schema; std::shared_ptr transport = instance_of; ReconnectMode reconnect_mode = ReconnectMode::testing; - std::shared_ptr custom_socket_provider = nullptr; }; explicit FLXSyncTestHarness(Config&& config) : m_test_session(make_app_from_server_schema(config.test_name, config.server_schema), config.transport, true, - config.reconnect_mode, config.custom_socket_provider) + config.reconnect_mode) , m_schema(std::move(config.server_schema.schema)) { } FLXSyncTestHarness(const std::string& test_name, ServerSchema server_schema = default_server_schema(), - std::shared_ptr transport = instance_of, - std::shared_ptr custom_socket_provider = nullptr) - : m_test_session(make_app_from_server_schema(test_name, server_schema), std::move(transport), true, - realm::ReconnectMode::normal, custom_socket_provider) + std::shared_ptr transport = instance_of) + : m_test_session(make_app_from_server_schema(test_name, server_schema), std::move(transport)) , m_schema(std::move(server_schema.schema)) { } diff --git a/test/object-store/sync/session/session.cpp b/test/object-store/sync/session/session.cpp index 4628458098c..e957f56f42a 100644 --- a/test/object-store/sync/session/session.cpp +++ b/test/object-store/sync/session/session.cpp @@ -257,62 +257,6 @@ TEST_CASE("SyncSession: close() API", "[sync]") { } } -TEST_CASE("SyncSession: pause()/resume() API", "[sync]") { - TestSyncManager init_sync_manager; - auto app = init_sync_manager.app(); - auto user = app->sync_manager()->get_user("close-api-tests-user", ENCODE_FAKE_JWT("fake_refresh_token"), - ENCODE_FAKE_JWT("fake_access_token"), "https://realm.example.org", - dummy_device_id); - - auto session = sync_session( - user, "/test-close-for-active", [](auto, auto) {}, SyncSessionStopPolicy::AfterChangesUploaded); - EventLoop::main().run_until([&] { - return sessions_are_active(*session); - }); - REQUIRE(sessions_are_active(*session)); - - SECTION("making the session inactive and then pausing it should end up in the paused state") { - session->force_close(); - EventLoop::main().run_until([&] { - return sessions_are_inactive(*session); - }); - REQUIRE(sessions_are_inactive(*session)); - - session->pause(); - EventLoop::main().run_until([&] { - return session->state() == SyncSession::State::Paused; - }); - REQUIRE(session->state() == SyncSession::State::Paused); - } - - SECTION("pausing from the active state should end up in the paused state") { - session->pause(); - EventLoop::main().run_until([&] { - return session->state() == SyncSession::State::Paused; - }); - REQUIRE(session->state() == SyncSession::State::Paused); - - // Pausing it again should be a no-op - session->pause(); - REQUIRE(session->state() == SyncSession::State::Paused); - - // "Logging out" the session should be a no-op. - session->force_close(); - REQUIRE(session->state() == SyncSession::State::Paused); - } - - // Reviving the session via revive_if_needed() should be a no-op. - session->revive_if_needed(); - REQUIRE(session->state() == SyncSession::State::Paused); - - // Only resume() can revive a paused session. - session->resume(); - EventLoop::main().run_until([&] { - return sessions_are_active(*session); - }); - REQUIRE(sessions_are_active(*session)); -} - TEST_CASE("SyncSession: shutdown_and_wait() API", "[sync]") { TestSyncManager init_sync_manager; auto app = init_sync_manager.app(); @@ -587,49 +531,6 @@ TEMPLATE_TEST_CASE("sync: stop policy behavior", "[sync]", RegularUser) } } -TEST_CASE("session restart", "[sync]") { - if (!EventLoop::has_implementation()) - return; - - TestSyncManager init_sync_manager({}, {false}); - auto& server = init_sync_manager.sync_server(); - auto app = init_sync_manager.app(); - Realm::Config config; - auto schema = Schema{ - {"object", - { - {"_id", PropertyType::Int, Property::IsPrimary{true}}, - {"value", PropertyType::Int}, - }}, - }; - - auto user = app->sync_manager()->get_user("userid", ENCODE_FAKE_JWT("fake_refresh_token"), - ENCODE_FAKE_JWT("fake_access_token"), dummy_auth_url, dummy_device_id); - auto session = sync_session( - user, "/test-restart", [&](auto, auto) {}, SyncSessionStopPolicy::AfterChangesUploaded, nullptr, schema, - &config); - - EventLoop::main().run_until([&] { - return sessions_are_active(*session); - }); - - server.start(); - - // Add an object so there's something to upload - auto realm = Realm::get_shared_realm(config); - TableRef table = ObjectStore::table_for_object_type(realm->read_group(), "object"); - realm->begin_transaction(); - table->create_object_with_primary_key(0); - realm->commit_transaction(); - - // Close the current session and start a new one - // The stop policy is ignored when closing the current session - session->restart_session(); - - REQUIRE(session->state() == SyncSession::State::Active); - REQUIRE(!wait_for_upload(*realm)); -} - TEST_CASE("sync: non-synced metadata table doesn't result in non-additive schema changes", "[sync]") { if (!EventLoop::has_implementation()) return; diff --git a/test/object-store/sync/session/session_util.hpp b/test/object-store/sync/session/session_util.hpp index eae1e7c9e19..0b910335792 100644 --- a/test/object-store/sync/session/session_util.hpp +++ b/test/object-store/sync/session/session_util.hpp @@ -43,8 +43,6 @@ struct StringMaker { return "Inactive"; case SyncSession::State::WaitingForAccessToken: return "WaitingForAccessToken"; - case SyncSession::State::Paused: - return "Paused"; default: return "Unknown"; } diff --git a/test/object-store/sync/sync_test_utils.cpp b/test/object-store/sync/sync_test_utils.cpp index 2aff9c35cc1..10b66cf82b2 100644 --- a/test/object-store/sync/sync_test_utils.cpp +++ b/test/object-store/sync/sync_test_utils.cpp @@ -330,8 +330,8 @@ struct FakeLocalClientReset : public TestClientReset { #if REALM_ENABLE_AUTH_TESTS -void wait_for_object_to_persist_to_atlas(std::shared_ptr user, const AppSession& app_session, - const std::string& schema_name, const bson::BsonDocument& filter_bson) +static void wait_for_object_to_persist(std::shared_ptr user, const AppSession& app_session, + const std::string& schema_name, const bson::BsonDocument& filter_bson) { // While at this point the object has been sync'd successfully, we must also // wait for it to appear in the backing database before terminating sync @@ -361,41 +361,6 @@ void wait_for_object_to_persist_to_atlas(std::shared_ptr user, const A std::chrono::minutes(15), std::chrono::milliseconds(500)); } -void trigger_client_reset(const AppSession& app_session) -{ - // cause a client reset by restarting the sync service - // this causes the server's sync history to be resynthesized - auto baas_sync_service = app_session.admin_api.get_sync_service(app_session.server_app_id); - auto baas_sync_config = app_session.admin_api.get_config(app_session.server_app_id, baas_sync_service); - - REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); - app_session.admin_api.disable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); - timed_sleeping_wait_for([&] { - return app_session.admin_api.is_sync_terminated(app_session.server_app_id); - }); - app_session.admin_api.enable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); - REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); - if (app_session.config.dev_mode_enabled) { // dev mode is not sticky across a reset - app_session.admin_api.set_development_mode_to(app_session.server_app_id, true); - } - - // In FLX sync, the server won't let you connect until the initial sync is complete. With PBS tho, we need - // to make sure we've actually copied all the data from atlas into the realm history before we do any of - // our remote changes. - if (!app_session.config.flx_sync_config) { - timed_sleeping_wait_for([&] { - return app_session.admin_api.is_initial_sync_complete(app_session.server_app_id); - }); - } -} - -void trigger_client_reset(const AppSession& app_session, const SharedRealm& realm) -{ - auto file_ident = SyncSession::OnlyForTesting::get_file_ident(*realm->sync_session()); - REQUIRE(file_ident.ident != 0); - app_session.admin_api.trigger_client_reset(app_session.server_app_id, file_ident.ident); -} - struct BaasClientReset : public TestClientReset { BaasClientReset(const Realm::Config& local_config, const Realm::Config& remote_config, TestAppSession& test_app_session) @@ -449,7 +414,10 @@ struct BaasClientReset : public TestClientReset { wait_for_upload(*realm); wait_for_download(*realm); - session->pause(); + wait_for_object_to_persist(m_local_config.sync_config->user, app_session, object_schema_name, + {{pk_col_name, m_pk_driving_reset}, {"value", last_synced_value}}); + + session->log_out(); realm->begin_transaction(); obj.set(col, 4); @@ -459,7 +427,24 @@ struct BaasClientReset : public TestClientReset { realm->commit_transaction(); } - trigger_client_reset(app_session, realm); + // cause a client reset by restarting the sync service + // this causes the server's sync history to be resynthesized + auto baas_sync_service = app_session.admin_api.get_sync_service(app_session.server_app_id); + auto baas_sync_config = app_session.admin_api.get_config(app_session.server_app_id, baas_sync_service); + REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); + app_session.admin_api.disable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); + timed_sleeping_wait_for([&] { + return app_session.admin_api.is_sync_terminated(app_session.server_app_id); + }); + app_session.admin_api.enable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); + REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); + if (app_session.config.dev_mode_enabled) { // dev mode is not sticky across a reset + app_session.admin_api.set_development_mode_to(app_session.server_app_id, true); + } + + timed_sleeping_wait_for([&] { + return app_session.admin_api.is_initial_sync_complete(app_session.server_app_id); + }); { auto realm2 = Realm::get_shared_realm(m_remote_config); @@ -501,7 +486,7 @@ struct BaasClientReset : public TestClientReset { } // Resuming sync on the first realm should now result in a client reset - session->resume(); + session->revive_if_needed(); if (m_on_post_local) { m_on_post_local(realm); } @@ -548,16 +533,32 @@ struct BaasFLXClientReset : public TestClientReset { auto ret = ObjectId::gen(); constexpr bool create_object = true; subscribe_to_object_by_id(realm, ret, create_object); + return ret; }(); - session->pause(); + wait_for_object_to_persist(m_local_config.sync_config->user, app_session, std::string(c_object_schema_name), + {{std::string(c_id_col_name), pk_of_added_object}}); + session->log_out(); if (m_make_local_changes) { m_make_local_changes(realm); } - trigger_client_reset(app_session, realm); + // cause a client reset by restarting the sync service + // this causes the server's sync history to be resynthesized + auto baas_sync_service = app_session.admin_api.get_sync_service(app_session.server_app_id); + auto baas_sync_config = app_session.admin_api.get_config(app_session.server_app_id, baas_sync_service); + REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); + app_session.admin_api.disable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); + timed_sleeping_wait_for([&] { + return app_session.admin_api.is_sync_terminated(app_session.server_app_id); + }); + app_session.admin_api.enable_sync(app_session.server_app_id, baas_sync_service.id, baas_sync_config); + REQUIRE(app_session.admin_api.is_sync_enabled(app_session.server_app_id)); + if (app_session.config.dev_mode_enabled) { // dev mode is not sticky across a reset + app_session.admin_api.set_development_mode_to(app_session.server_app_id, true); + } { auto realm2 = Realm::get_shared_realm(m_remote_config); @@ -592,7 +593,7 @@ struct BaasFLXClientReset : public TestClientReset { } // Resuming sync on the first realm should now result in a client reset - session->resume(); + session->revive_if_needed(); if (m_on_post_local) { m_on_post_local(realm); } @@ -617,13 +618,12 @@ struct BaasFLXClientReset : public TestClientReset { Query query_for_added_object = table->where().equal(id_col, pk); mut_subs.insert_or_assign(query_for_added_object); auto subs = std::move(mut_subs).commit(); - subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get(); if (create_object) { realm->begin_transaction(); table->create_object_with_primary_key(pk, {{str_col, "initial value"}}); realm->commit_transaction(); } - wait_for_upload(*realm); + subs.get_state_change_notification(sync::SubscriptionSet::State::Complete).get(); } void load_initial_data(SharedRealm realm) diff --git a/test/object-store/sync/sync_test_utils.hpp b/test/object-store/sync/sync_test_utils.hpp index 7fe0cf1c2fe..e0fc7ca775f 100644 --- a/test/object-store/sync/sync_test_utils.hpp +++ b/test/object-store/sync/sync_test_utils.hpp @@ -190,12 +190,6 @@ std::unique_ptr make_baas_client_reset(const Realm::Config& loc std::unique_ptr make_baas_flx_client_reset(const Realm::Config& local_config, const Realm::Config& remote_config, const TestAppSession& test_app_session); - -void wait_for_object_to_persist_to_atlas(std::shared_ptr user, const AppSession& app_session, - const std::string& schema_name, const bson::BsonDocument& filter_bson); - -void trigger_client_reset(const AppSession& app_session); -void trigger_client_reset(const AppSession& app_session, const SharedRealm& realm); #endif // REALM_ENABLE_AUTH_TESTS #endif // REALM_ENABLE_SYNC diff --git a/test/object-store/util/baas_admin_api.cpp b/test/object-store/util/baas_admin_api.cpp index c361c160c7b..6fd30c8188a 100644 --- a/test/object-store/util/baas_admin_api.cpp +++ b/test/object-store/util/baas_admin_api.cpp @@ -582,12 +582,6 @@ AdminAPISession::Service AdminAPISession::get_sync_service(const std::string& ap return *sync_service; } -void AdminAPISession::trigger_client_reset(const std::string& app_id, int64_t file_ident) const -{ - auto endpoint = apps(APIFamily::Private)[app_id]["sync"]["force_reset"]; - endpoint.put_json(nlohmann::json{{"file_ident", file_ident}}); -} - static nlohmann::json convert_config(AdminAPISession::ServiceConfig config) { if (config.mode == AdminAPISession::ServiceConfig::SyncMode::Flexible) { @@ -722,17 +716,9 @@ bool AdminAPISession::is_initial_sync_complete(const std::string& app_id) const return false; } -AdminAPIEndpoint AdminAPISession::apps(APIFamily family) const +AdminAPIEndpoint AdminAPISession::apps() const { - switch (family) { - case APIFamily::Admin: - return AdminAPIEndpoint(util::format("%1/api/admin/v3.0/groups/%2/apps", m_base_url, m_group_id), - m_access_token); - case APIFamily::Private: - return AdminAPIEndpoint(util::format("%1/api/private/v1.0/groups/%2/apps", m_base_url, m_group_id), - m_access_token); - } - REALM_UNREACHABLE(); + return AdminAPIEndpoint(util::format("%1/api/admin/v3.0/groups/%2/apps", m_base_url, m_group_id), m_access_token); } AppCreateConfig default_app_config(const std::string& base_url) diff --git a/test/object-store/util/baas_admin_api.hpp b/test/object-store/util/baas_admin_api.hpp index d47d0acc08f..6ff20f1d6eb 100644 --- a/test/object-store/util/baas_admin_api.hpp +++ b/test/object-store/util/baas_admin_api.hpp @@ -67,15 +67,13 @@ class AdminAPISession { static AdminAPISession login(const std::string& base_url, const std::string& username, const std::string& password); - enum class APIFamily { Admin, Private }; - AdminAPIEndpoint apps(APIFamily family = APIFamily::Admin) const; + AdminAPIEndpoint apps() const; void revoke_user_sessions(const std::string& user_id, const std::string& app_id) const; void disable_user_sessions(const std::string& user_id, const std::string& app_id) const; void enable_user_sessions(const std::string& user_id, const std::string& app_id) const; bool verify_access_token(const std::string& access_token, const std::string& app_id) const; void set_development_mode_to(const std::string& app_id, bool enable) const; void delete_app(const std::string& app_id) const; - void trigger_client_reset(const std::string& app_id, int64_t file_ident) const; struct Service { std::string id; diff --git a/test/object-store/util/test_file.cpp b/test/object-store/util/test_file.cpp index bbcb955fdc8..5b41dac967e 100644 --- a/test/object-store/util/test_file.cpp +++ b/test/object-store/util/test_file.cpp @@ -139,19 +139,6 @@ SyncTestFile::SyncTestFile(std::shared_ptr user, bson::Bson partition, schema_mode = SchemaMode::AdditiveExplicit; } -SyncTestFile::SyncTestFile(std::shared_ptr user, bson::Bson partition, - realm::util::Optional schema, - std::function&& error_handler) -{ - REALM_ASSERT(user); - sync_config = std::make_shared(user, partition); - sync_config->stop_policy = SyncSessionStopPolicy::Immediately; - sync_config->error_handler = std::move(error_handler); - schema_version = 1; - this->schema = std::move(schema); - schema_mode = SchemaMode::AdditiveExplicit; -} - SyncTestFile::SyncTestFile(std::shared_ptr user, realm::Schema _schema, SyncConfig::FLXSyncEnabled) { REALM_ASSERT(user); @@ -180,7 +167,8 @@ SyncServer::SyncServer(const SyncServer::Config& config) using namespace std::literals::chrono_literals; #if TEST_ENABLE_SYNC_LOGGING - auto logger = new util::StderrLogger(realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL); + auto logger = new util::StderrLogger(); + logger->set_level_threshold(realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL); m_logger.reset(logger); #else // Logging is disabled, use a NullLogger to prevent printing anything @@ -310,8 +298,7 @@ TestAppSession::TestAppSession() TestAppSession::TestAppSession(AppSession session, std::shared_ptr custom_transport, - DeleteApp delete_app, ReconnectMode reconnect_mode, - std::shared_ptr custom_socket_provider) + DeleteApp delete_app, ReconnectMode reconnect_mode) : m_app_session(std::make_unique(session)) , m_base_file_path(util::make_temp_dir() + random_string(10)) , m_delete_app(delete_app) @@ -328,7 +315,6 @@ TestAppSession::TestAppSession(AppSession session, sc_config.log_level = realm::util::Logger::Level::TEST_ENABLE_SYNC_LOGGING_LEVEL; sc_config.metadata_mode = realm::SyncManager::MetadataMode::NoEncryption; sc_config.reconnect_mode = reconnect_mode; - sc_config.socket_provider = custom_socket_provider; m_app = app::App::get_uncached_app(app_config, sc_config); diff --git a/test/object-store/util/test_file.hpp b/test/object-store/util/test_file.hpp index f82a73b6b7b..acd0f8feb2b 100644 --- a/test/object-store/util/test_file.hpp +++ b/test/object-store/util/test_file.hpp @@ -169,9 +169,6 @@ struct SyncTestFile : TestFile { std::string user_name = "test"); SyncTestFile(std::shared_ptr user, realm::bson::Bson partition, realm::util::Optional schema = realm::util::none); - SyncTestFile(std::shared_ptr user, realm::bson::Bson partition, - realm::util::Optional schema, - std::function&& error_handler); SyncTestFile(std::shared_ptr app, realm::bson::Bson partition, realm::Schema schema); SyncTestFile(std::shared_ptr user, realm::Schema schema, realm::SyncConfig::FLXSyncEnabled); }; @@ -182,8 +179,7 @@ class TestAppSession { public: TestAppSession(); TestAppSession(realm::AppSession, std::shared_ptr = nullptr, - DeleteApp = true, realm::ReconnectMode reconnect_mode = realm::ReconnectMode::normal, - std::shared_ptr custom_socket_provider = nullptr); + DeleteApp = true, realm::ReconnectMode reconnect_mode = realm::ReconnectMode::normal); ~TestAppSession(); std::shared_ptr app() const noexcept diff --git a/test/realm-fuzzer/CMakeLists.txt b/test/realm-fuzzer/CMakeLists.txt deleted file mode 100644 index 03353ff0545..00000000000 --- a/test/realm-fuzzer/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -set(TEST_AFL_SOURCES - afl_runner.cpp - fuzz_engine.cpp - fuzz_object.cpp - fuzz_configurator.cpp -) # TEST_AFL_SOURCES_OBJECT_STORE - -set(TEST_LIBFUZZER_SOURCES - libfuzzer_runner.cpp - fuzz_engine.cpp - fuzz_object.cpp - fuzz_configurator.cpp -) # TEST_LIBFUZZER_SOURCES_OBJECT_STORE - -file(GLOB FUZZER_RUN_SCRIPTS - "scripts/start_fuzz_afl.sh" - "scripts/start_lib_fuzzer.sh") - -file(COPY ${FUZZER_RUN_SCRIPTS} - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file(GLOB AFL_SEEDS "testcases/*") -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testcases) -file(COPY ${AFL_SEEDS} - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/testcases) - -add_executable(realm-afl++ ${TEST_AFL_SOURCES}) -target_link_libraries(realm-afl++ TestUtil ObjectStore) - -if(REALM_LIBFUZZER) - if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - add_executable(realm-libfuzz ${TEST_LIBFUZZER_SOURCES}) - target_link_libraries(realm-libfuzz TestUtil ObjectStore) - endif() -endif() - -# on Apple platforms we use the built-in CFRunLoop -# everywhere else it's libuv, except UWP where it doesn't build -if(NOT APPLE AND NOT WINDOWS_STORE) - target_link_libraries(realm-afl++ uv_a) - if(REALM_LIBFUZZER) - if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - target_link_libraries(realm-libfuzz uv_a) - endif() - endif() -endif() \ No newline at end of file diff --git a/test/realm-fuzzer/README.md b/test/realm-fuzzer/README.md deleted file mode 100644 index b0070c1c7d2..00000000000 --- a/test/realm-fuzzer/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# The Fuzz Framework project - -This project is an attempt to put together all the small fuzzers we have already scattered around the code. -There are two goals: - 1. To be able to run all the fuzzers, collect crashes reports and fix possible bugs that the fuzzer might find. - 2. To be able to replace libfuzzer with google fuzz test (https://github.com/google/fuzztest) at some point. - -AFL++ support is not dropped yet, but since we want to integrate things inside evergreen and follow the same approach we implement for address/thread sanitazer we prefer to use libfuzzer and clang. -## Prerequisites - -In case you want to use AFL++, then you should install the latest version of the American Fuzzy Lop ++ (AFL++). -Please use this quick guide: https://aflplus.plus/building/ it requires llvm >= 9.0. - -For using libfuzzer, the only pre-requisite is having a recent version of clang. -## Running -If you don't want to build manually, you can skip this section and jump to the `Scripts` section. \ -Run the fuzzer via AFL++: - -``` -cd -mkdir build -cd build -cmake -D CMAKE_BUILD_TYPE=${build_mode} - -D CMAKE_C_COMPILER=afl-cc - -D CMAKE_CXX_COMPILER=afl-c++ - -D REALM_ENABLE_ENCRYPTION=OFF - -G Ninja - .. -cmake --build . --target realm-afl++ -afl-fuzz -t "$time_out" - -m "$memory" - -i "${ROOT_DIR}/test/fuzzy_object_store/testcases" - -o "${FINDINGS_DIR}" - realm-afl++ @@ -``` - -Run the fuzzer via libFuzzer (only with Clang) -``` -cd -mkdir build -cd build -cmake -D REALM_LIBFUZZER=ON - -D CMAKE_BUILD_TYPE=${build_mode} - -D CMAKE_C_COMPILER=clang - -D CMAKE_CXX_COMPILER=clang++ - -D REALM_ENABLE_ENCRYPTION=OFF - -G Ninja - .. -cmake --build . --target realm-libfuzz -./realm_libfuzz -``` - -## Scripts - -`sh start_fuzz_afl.sh` -Builds `realm-core` and `object-store` in `Debug` mode using the afl++ compiler `afl-cc` and starts 1 instance of `afl-fuzz`. -It expects `AFLPlusPlus` to be installed in your system and in general added to your `PATH`. -Optionally, the following arguments can be passed to the script: -1) `` the number of fuzzers to launch (by default 1). -2) `` either `Release` or `Debug`. - -`sh start_lib_fuzzer.sh` -Builds `realm-core` and `object-store` in `Debug` mode using the clang compiler and starts `realm-libfuzz`. -Optionally, the following arguments can be passed to the script: -1) `` either `Release` or `Debug`. -2) `` essentially initial set of inputs for improving fuzzer efficiency. - -## See Also - -[AFL++ github](https://github.com/AFLplusplus/AFLplusplus) \ -[LibFuzzer](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) \ -[Google Fuzz Test](https://github.com/google/fuzztest) diff --git a/test/realm-fuzzer/afl_runner.cpp b/test/realm-fuzzer/afl_runner.cpp deleted file mode 100644 index 88def06330a..00000000000 --- a/test/realm-fuzzer/afl_runner.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#include "fuzz_engine.hpp" - -int main(int argc, const char* argv[]) -{ - FuzzEngine fuzz_engine; - bool enable_logging = false; - std::string path = "realm-afl.txt"; - size_t input_index = 0; - for (size_t i = 0; i < (size_t)argc; ++i) { - if (strcmp(argv[i], "--log") == 0) { - enable_logging = true; - } - else { - input_index = i; - } - } - return fuzz_engine.run_fuzzer(argv[input_index], "realm_afl", enable_logging, path); -} \ No newline at end of file diff --git a/test/realm-fuzzer/fuzz_configurator.cpp b/test/realm-fuzzer/fuzz_configurator.cpp deleted file mode 100644 index e080e77f1c1..00000000000 --- a/test/realm-fuzzer/fuzz_configurator.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ -#include "fuzz_configurator.hpp" -#include "fuzz_object.hpp" -#include "../util/test_path.hpp" - -FuzzConfigurator::FuzzConfigurator(FuzzObject& fuzzer, const std::string& input, bool use_input_file, - const std::string& name) - : m_used_input_file(use_input_file) - , m_fuzzer(fuzzer) - , m_fuzz_name(name) -{ - realm::disable_sync_to_disk(); - init(input); - setup_realm_config(); -} - -void FuzzConfigurator::setup_realm_config() -{ - m_config.path = m_path; - m_config.schema_version = 0; - if (m_use_encryption) { - const char* key = m_fuzzer.get_encryption_key(); - const char* i = key; - while (*i != '\0') { - m_config.encryption_key.push_back(*i); - i++; - } - } -} - -const realm::Realm::Config& FuzzConfigurator::get_config() const -{ - return m_config; -} - -FuzzObject& FuzzConfigurator::get_fuzzer() -{ - return m_fuzzer; -} - -const std::string& FuzzConfigurator::get_realm_path() const -{ - return m_path; -} - -FuzzLog& FuzzConfigurator::get_logger() -{ - return m_log; -} - -State& FuzzConfigurator::get_state() -{ - return m_state; -} - -void FuzzConfigurator::init(const std::string& input) -{ - std::string db_name = "fuzz-test"; - realm::test_util::RealmPathInfo test_context{db_name}; - SHARED_GROUP_TEST_PATH(path); - m_path = path.c_str(); - if (m_used_input_file) { - std::ifstream in(input, std::ios::in | std::ios::binary); - if (!in.is_open()) { - std::cerr << "Could not open file for reading: " << input << "\n"; - throw; - } - std::string contents((std::istreambuf_iterator(in)), (std::istreambuf_iterator())); - set_state(contents); - } - else { - set_state(input); - } -} - -void FuzzConfigurator::set_state(const std::string& input) -{ - m_state = State{input, 0}; - m_use_encryption = m_fuzzer.get_next_token(m_state) % 2 == 0; -} - -void FuzzConfigurator::print_cnf() -{ - m_log << "// Fuzzer: " << m_fuzz_name << "\n"; - m_log << "// Test case generated in " REALM_VER_CHUNK " on " << m_fuzzer.get_current_time_stamp() << ".\n"; - m_log << "// REALM_MAX_BPNODE_SIZE is " << REALM_MAX_BPNODE_SIZE << "\n"; - m_log << "// ----------------------------------------------------------------------\n"; - const auto& printable_key = - !m_use_encryption ? "nullptr" : std::string("\"") + m_config.encryption_key.data() + "\""; - m_log << "// const char* key = " << printable_key << ";\n"; - m_log << "\n"; -} \ No newline at end of file diff --git a/test/realm-fuzzer/fuzz_configurator.hpp b/test/realm-fuzzer/fuzz_configurator.hpp deleted file mode 100644 index 33c3203a181..00000000000 --- a/test/realm-fuzzer/fuzz_configurator.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ -#ifndef FUZZ_CONFIG_HPP -#define FUZZ_CONFIG_HPP - -#include "util.hpp" -#include "fuzz_logger.hpp" -#include -#include -#include - -class FuzzObject; -class FuzzConfigurator { -public: - FuzzConfigurator(FuzzObject& fuzzer, const std::string& input, bool use_input_file, const std::string& name); - const realm::Realm::Config& get_config() const; - FuzzObject& get_fuzzer(); - const std::string& get_realm_path() const; - FuzzLog& get_logger(); - State& get_state(); - void set_state(const std::string& input); - void print_cnf(); - -private: - void init(const std::string&); - void setup_realm_config(); - - realm::Realm::Config m_config; - std::string m_path; - FuzzLog m_log; - bool m_use_encryption{false}; - bool m_used_input_file{false}; - FuzzObject& m_fuzzer; - State m_state; - std::string m_fuzz_name; -}; -#endif \ No newline at end of file diff --git a/test/realm-fuzzer/fuzz_engine.cpp b/test/realm-fuzzer/fuzz_engine.cpp deleted file mode 100644 index a4c4d4edd7c..00000000000 --- a/test/realm-fuzzer/fuzz_engine.cpp +++ /dev/null @@ -1,182 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#include "fuzz_engine.hpp" -#include "fuzz_configurator.hpp" -#include "fuzz_object.hpp" -#include "util.hpp" - -#include -#include -#include -#include - -using namespace realm; -const size_t max_tables = REALM_MAX_BPNODE_SIZE * 10; - -const char hex_digits[] = "0123456789abcdefgh"; -char hex_buffer[3]; - -static const char* to_hex(char c) -{ - hex_buffer[0] = hex_digits[(c >> 4) & 0x0f]; - hex_buffer[1] = hex_digits[c & 0x0f]; - return hex_buffer; -} - -int FuzzEngine::run_fuzzer(const std::string& input, const std::string& name, bool enable_logging, - const std::string& path) -{ - auto configure = [&](auto fuzzer) { - try { - FuzzConfigurator cnf(fuzzer, input, false, name); - if (enable_logging) { - cnf.get_logger().enable_logging(path); - cnf.print_cnf(); - } - return cnf; - } - catch (const EndOfFile& e) { - throw std::runtime_error{"Realm cnf is invalid"}; - } - }; - - try { - FuzzObject fuzzer; - auto cnf = configure(fuzzer); - do_fuzz(cnf); - } - catch (const EndOfFile&) { - } - return 0; -} - -void FuzzEngine::do_fuzz(FuzzConfigurator& cnf) -{ - const auto path = cnf.get_realm_path(); - auto& log = cnf.get_logger(); - auto& state = cnf.get_state(); - auto& fuzzer = cnf.get_fuzzer(); - auto shared_realm = Realm::get_shared_realm(cnf.get_config()); - std::vector table_views; - - log << "Start fuzzing with state = "; - for (auto c : state.str) { - log << to_hex(c) << " "; - } - log << "\n"; - - auto begin_write = [&log](SharedRealm shared_realm) -> Group& { - log << "begin_write() - check : shared_realm->is_in_transaction()\n"; - if (!shared_realm->is_in_transaction() && !shared_realm->is_in_async_transaction()) { - log << "begin_write() - open transaction : shared_realm->begin_transaction()\n"; - try { - shared_realm->begin_transaction(); - } - catch (std::exception& e) { - log << e.what() << "\n"; - throw; - } - } - log << "begin_write() - return shared_realm->read_group();\n"; - return shared_realm->read_group(); - }; - - int iteration = 0; - - for (;;) { - char instr = fuzzer.get_next_token(state) % Count; - iteration++; - log << "Iteration: " << iteration << ". fuzz with command: " << std::to_string(instr) << "\n"; - - try { - Group& group = begin_write(shared_realm); - if (instr == Add_Table && group.size() < max_tables) { - fuzzer.create_table(group, log); - } - else if (instr == Remove_Table && group.size() > 0) { - fuzzer.remove_table(group, log, state); - } - else if (instr == Clear_Table && group.size() > 0) { - fuzzer.clear_table(group, log, state); - } - else if (instr == Create_Object && group.size() > 0) { - fuzzer.create_object(group, log, state); - } - else if (instr == Add_Column && group.size() > 0) { - fuzzer.add_column(group, log, state); - } - else if (instr == Remove_Column && group.size() > 0) { - fuzzer.remove_column(group, log, state); - } - else if (instr == Get_All_Column_Names && group.size() > 0) { - fuzzer.get_all_column_names(group, log); - } - else if (instr == Rename_Column && group.size() > 0) { - fuzzer.rename_column(group, log, state); - } - else if (instr == Add_Search_Index && group.size() > 0) { - fuzzer.add_search_index(group, log, state); - } - else if (instr == Remove_Search_Index && group.size() > 0) { - fuzzer.remove_search_index(group, log, state); - } - else if (instr == Add_Column_Link && group.size() >= 1) { - fuzzer.add_column_link(group, log, state); - } - else if (instr == Add_Column_Link_List && group.size() >= 2) { - fuzzer.add_column_link_list(group, log, state); - } - else if (instr == Instruction::Set && group.size() > 0) { - fuzzer.set_obj(group, log, state); - } - else if (instr == Remove_Object && group.size() > 0) { - fuzzer.remove_obj(group, log, state); - } - else if (instr == Remove_Recursive && group.size() > 0) { - fuzzer.remove_recursive(group, log, state); - } - else if (instr == Enumerate_Column && group.size() > 0) { - fuzzer.enumerate_column(group, log, state); - } - else if (instr == Commit) { - fuzzer.commit(shared_realm, log); - } - else if (instr == Rollback) { - fuzzer.rollback(shared_realm, group, log); - } - else if (instr == Advance) { - fuzzer.advance(shared_realm, log); - } - else if (instr == Close_And_Reopen) { - fuzzer.close_and_reopen(shared_realm, log, cnf.get_config()); - } - else if (instr == Create_Table_View && group.size() > 0) { - fuzzer.create_table_view(group, log, state, table_views); - } - else if (instr == Compact) { - } - else if (instr == Is_Null && group.size() > 0) { - fuzzer.check_null(group, log, state); - } - } - catch (const std::exception& e) { - log << "\nException thrown during execution:\n" << e.what() << "\n"; - } - } -} diff --git a/test/realm-fuzzer/fuzz_engine.hpp b/test/realm-fuzzer/fuzz_engine.hpp deleted file mode 100644 index 16b15fc9f5b..00000000000 --- a/test/realm-fuzzer/fuzz_engine.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#ifndef FUZZ_ENGINE_HPP -#define FUZZ_ENGINE_HPP - -#include -#include -#include -#include -#include -#include -#if REALM_USE_UV -#include -#endif - -class FuzzConfigurator; -class FuzzEngine { -public: - int run_fuzzer(const std::string& input, const std::string& name, bool = false, const std::string& = ""); - -private: - void do_fuzz(FuzzConfigurator&); -}; - -#endif diff --git a/test/realm-fuzzer/fuzz_logger.hpp b/test/realm-fuzzer/fuzz_logger.hpp deleted file mode 100644 index a0b94d1c030..00000000000 --- a/test/realm-fuzzer/fuzz_logger.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ -#ifndef FUZZ_LOGGER_HPP -#define FUZZ_LOGGER_HPP -#include -#include - -class FuzzLog { -public: - FuzzLog() = default; - - template - FuzzLog& operator<<(const T& v) - { - if (m_active) { - m_out << v; - m_out.flush(); - } - return *this; - } - - void enable_logging(const std::string& path) - { - m_out.open(path, std::ios::out); - m_active = true; - } - -private: - std::fstream m_out; - bool m_active{false}; -}; - -#endif \ No newline at end of file diff --git a/test/realm-fuzzer/fuzz_object.cpp b/test/realm-fuzzer/fuzz_object.cpp deleted file mode 100644 index d2b07247c6b..00000000000 --- a/test/realm-fuzzer/fuzz_object.cpp +++ /dev/null @@ -1,547 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#include "fuzz_object.hpp" -#include "fuzz_logger.hpp" -#include "util.hpp" - -#include -#include -#include -#include -#include -#include - -#include -#include - -using namespace realm; -using namespace realm::util; - -// Max number of rows in a table. Overridden only by create_object() and only in the case where -// max_rows is not exceeded *prior* to executing add_empty_row. -const size_t max_rows = 100000; -const size_t add_empty_row_max = REALM_MAX_BPNODE_SIZE * REALM_MAX_BPNODE_SIZE + 1000; - -unsigned char FuzzObject::get_next_token(State& s) const -{ - if (s.pos == s.str.size() || s.str.empty()) { - throw EndOfFile{}; - } - return s.str[s.pos++]; -} - -void FuzzObject::create_table(Group& group, FuzzLog& log) -{ - log << "FuzzObject::create_table();\n"; - std::string name = create_table_name(); - log << "group.add_table(\"" << name << "\");\n"; - group.add_table(name); -} - -void FuzzObject::remove_table(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::remove_table();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - log << "try { group.remove_table(" << table_key - << "); }" - " catch (const CrossTableLinkTarget&) { }\n"; - group.remove_table(table_key); -} - -void FuzzObject::clear_table(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::clear_table();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - log << "group.get_table(" << table_key << ")->clear();\n"; - group.get_table(table_key)->clear(); -} - -void FuzzObject::create_object(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::create_object();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - size_t num_rows = get_next_token(s); - if (group.get_table(table_key)->size() + num_rows < max_rows) { - log << "{ std::vector keys; wt->get_table(" << table_key << ")->create_objects(" - << num_rows % add_empty_row_max << ", keys); }\n"; - std::vector keys; - group.get_table(table_key)->create_objects(num_rows % add_empty_row_max, keys); - } -} - -void FuzzObject::add_column(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::add_column();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - DataType type = get_type(get_next_token(s)); - std::string name = create_column_name(type); - // Mixed cannot be nullable. For other types, chose nullability randomly - bool nullable = (get_next_token(s) % 2 == 0); - log << "group.get_table(" << table_key << ")->add_column(DataType(" << int(type) << "), \"" << name << "\", " - << (nullable ? "true" : "false") << ");"; - auto col = group.get_table(table_key)->add_column(type, name, nullable); - log << " // -> " << col << "\n"; -} - -void FuzzObject::remove_column(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::remove_column();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto column_keys = t->get_column_keys(); - if (!column_keys.empty()) { - ColKey col = column_keys[get_next_token(s) % column_keys.size()]; - log << "group.get_table(" << table_key << ")->remove_column(" << col << ");\n"; - t->remove_column(col); - } -} - -void FuzzObject::rename_column(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::rename_column();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto column_keys = t->get_column_keys(); - if (!column_keys.empty()) { - ColKey col = column_keys[get_next_token(s) % column_keys.size()]; - std::string name = create_column_name(t->get_column_type(col)); - log << "group.get_table(" << table_key << ")->rename_column(" << col << ", \"" << name << "\");\n"; - t->rename_column(col, name); - } -} - -void FuzzObject::add_search_index(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::add_search_index();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto column_keys = t->get_column_keys(); - if (!column_keys.empty()) { - ColKey col = column_keys[get_next_token(s) % column_keys.size()]; - bool supports_search_index = StringIndex::type_supported(t->get_column_type(col)); - - if (supports_search_index) { - log << "group.get_table(" << table_key << ")->add_search_index(" << col << ");\n"; - t->add_search_index(col); - } - } -} - -void FuzzObject::remove_search_index(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::remove_search_index();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto column_keys = t->get_column_keys(); - if (!column_keys.empty()) { - ColKey col = column_keys[get_next_token(s) % column_keys.size()]; - // We don't need to check if the column is of a type that is indexable or if it has index on or off - // because Realm will just do a no-op at worst (no exception or assert). - log << "group.get_table(" << table_key << ")->remove_search_index(" << col << ");\n"; - t->remove_search_index(col); - } -} - -void FuzzObject::add_column_link(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::add_column_link();\n"; - TableKey table_key_1 = group.get_table_keys()[get_next_token(s) % group.size()]; - TableKey table_key_2 = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t1 = group.get_table(table_key_1); - TableRef t2 = group.get_table(table_key_2); - std::string name = create_column_name(type_Link); - log << "group.get_table(" << table_key_1 << ")->add_column_link(type_Link, \"" << name << "\", *group->get_table(" - << table_key_2 << "));"; - auto col = t1->add_column(*t2, name); - log << " // -> " << col << "\n"; -} - -void FuzzObject::add_column_link_list(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::add_column_link_list();\n"; - TableKey table_key_1 = group.get_table_keys()[get_next_token(s) % group.size()]; - TableKey table_key_2 = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t1 = group.get_table(table_key_1); - TableRef t2 = group.get_table(table_key_2); - std::string name = create_column_name(type_LinkList); - log << "group.get_table(" << table_key_1 << ")->add_column_link(type_LinkList, \"" << name - << "\", group.get_table(" << table_key_2 << "));"; - auto col = t1->add_column_list(*t2, name); - log << " // -> " << col << "\n"; -} - -void FuzzObject::set_obj(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::set_obj();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto all_col_keys = t->get_column_keys(); - if (!all_col_keys.empty() && t->size() > 0) { - ColKey col = all_col_keys[get_next_token(s) % all_col_keys.size()]; - size_t row = get_next_token(s) % t->size(); - DataType type = t->get_column_type(col); - Obj obj = t->get_object(row); - log << "{\nObj obj = group.get_table(" << table_key << ")->get_object(" << row << ");\n"; - - // With equal probability, either set to null or to a value - if (get_next_token(s) % 2 == 0 && t->is_nullable(col)) { - if (type == type_Link) { - log << "obj.set(" << col << ", null_key);\n"; - obj.set(col, null_key); - } - else { - log << "obj.set_null(" << col << ");\n"; - obj.set_null(col); - } - } - else { - if (type == type_String) { - std::string str = create_string(get_next_token(s)); - log << "obj.set(" << col << ", \"" << str << "\");\n"; - obj.set(col, StringData(str)); - } - else if (type == type_Binary) { - std::string str = create_string(get_next_token(s)); - log << "obj.set(" << col << ", BinaryData{\"" << str << "\", " << str.size() << "});\n"; - obj.set(col, BinaryData(str)); - } - else if (type == type_Int) { - bool add_int = get_next_token(s) % 2 == 0; - int64_t value = get_int64(s); - if (add_int) { - log << "try { obj.add_int(" << col << ", " << value - << "); } catch (const LogicError& le) { CHECK(le.kind() == " - "LogicError::illegal_combination); }\n"; - try { - obj.add_int(col, value); - } - catch (const LogicError& le) { - if (le.kind() != LogicError::illegal_combination) { - throw; - } - } - } - else { - log << "obj.set(" << col << ", " << value << ");\n"; - obj.set(col, value); - } - } - else if (type == type_Bool) { - bool value = get_next_token(s) % 2 == 0; - log << "obj.set(" << col << ", " << (value ? "true" : "false") << ");\n"; - obj.set(col, value); - } - else if (type == type_Float) { - float value = get_next_token(s); - log << "obj.set(" << col << ", " << value << ");\n"; - obj.set(col, value); - } - else if (type == type_Double) { - double value = get_next_token(s); - log << "obj.set(" << col << ", " << value << ");\n"; - obj.set(col, value); - } - else if (type == type_Link) { - TableRef target = t->get_link_target(col); - if (target->size() > 0) { - ObjKey target_key = target->get_object(get_next_token(s) % target->size()).get_key(); - log << "obj.set(" << col << ", " << target_key << ");\n"; - obj.set(col, target_key); - } - } - else if (type == type_LinkList) { - TableRef target = t->get_link_target(col); - if (target->size() > 0) { - LnkLst links = obj.get_linklist(col); - ObjKey target_key = target->get_object(get_next_token(s) % target->size()).get_key(); - // either add or set, 50/50 probability - if (links.size() > 0 && get_next_token(s) > 128) { - size_t linklist_row = get_next_token(s) % links.size(); - log << "obj.get_linklist(" << col << ")->set(" << linklist_row << ", " << target_key - << ");\n"; - links.set(linklist_row, target_key); - } - else { - log << "obj.get_linklist(" << col << ")->add(" << target_key << ");\n"; - links.add(target_key); - } - } - } - else if (type == type_Timestamp) { - std::pair values = get_timestamp_values(s); - Timestamp value{values.first, values.second}; - log << "obj.set(" << col << ", " << value << ");\n"; - obj.set(col, value); - } - } - log << "}\n"; - } - else { - log << "table " << table_key << " has size = " << t->size() - << " and get_column_keys size = " << all_col_keys.size() << "\n"; - } -} - -void FuzzObject::remove_obj(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::remove_obj();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - if (t->size() > 0) { - ObjKey key = t->get_object(get_next_token(s) % t->size()).get_key(); - log << "group.get_table(" << table_key << ")->remove_object(" << key << ");\n"; - t->remove_object(key); - } -} - -void FuzzObject::remove_recursive(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::remove_recursive();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - if (t->size() > 0) { - ObjKey key = t->get_object(get_next_token(s) % t->size()).get_key(); - log << "group.get_table(" << table_key << ")->remove_object_recursive(" << key << ");\n"; - t->remove_object_recursive(key); - } -} - -void FuzzObject::enumerate_column(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::enumerate_column();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - auto all_col_keys = t->get_column_keys(); - if (!all_col_keys.empty()) { - size_t ndx = get_next_token(s) % all_col_keys.size(); - ColKey col = all_col_keys[ndx]; - log << "group.get_table(" << table_key << ")->enumerate_string_column(" << col << ");\n"; - group.get_table(table_key)->enumerate_string_column(col); - } -} - -void FuzzObject::get_all_column_names(Group& group, FuzzLog& log) -{ - log << "FuzzObject::get_all_column_names();\n"; - for (auto table_key : group.get_table_keys()) { - TableRef t = group.get_table(table_key); - auto all_col_keys = t->get_column_keys(); - for (auto col : all_col_keys) { - StringData col_name = t->get_column_name(col); - static_cast(col_name); - } - } -} - -void FuzzObject::commit(SharedRealm shared_realm, FuzzLog& log) -{ - log << "FuzzObject::commit();\n"; - log << "FuzzObject::commit() - shared_realm->is_in_transaction();\n"; - if (shared_realm->is_in_transaction()) { - log << "FuzzObject::commit() - shared_realm->commit_transaction();\n"; - shared_realm->commit_transaction(); - auto& group = shared_realm->read_group(); - REALM_DO_IF_VERIFY(log, group.verify()); - } -} - -void FuzzObject::rollback(SharedRealm shared_realm, Group& group, FuzzLog& log) -{ - log << "FuzzObject::rollback()\n"; - if (!shared_realm->is_in_async_transaction() && !shared_realm->is_in_transaction()) { - shared_realm->begin_transaction(); - REALM_DO_IF_VERIFY(log, group.verify()); - log << "shared_realm->cancel_transaction();\n"; - shared_realm->cancel_transaction(); - REALM_DO_IF_VERIFY(log, shared_realm->read_group().verify()); - } -} - -void FuzzObject::advance(realm::SharedRealm shared_realm, FuzzLog& log) -{ - log << "FuzzObject::advance();\n"; - shared_realm->notify(); -} - -void FuzzObject::close_and_reopen(SharedRealm& shared_realm, FuzzLog& log, const Realm::Config& config) -{ - log << "Open/close realm\n"; - shared_realm->close(); - shared_realm.reset(); - shared_realm = Realm::get_shared_realm(config); - log << "Verify group after realm got reopened\n"; - auto& group = shared_realm->read_group(); - REALM_DO_IF_VERIFY(log, group.verify()); -} - -void FuzzObject::create_table_view(Group& group, FuzzLog& log, State& s, std::vector& table_views) -{ - log << "FuzzObject::create_table_view();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - log << "table_views.push_back(wt->get_table(" << table_key << ")->where().find_all());\n"; - TableView tv = t->where().find_all(); - table_views.push_back(tv); -} - -void FuzzObject::check_null(Group& group, FuzzLog& log, State& s) -{ - log << "FuzzObject::check_null();\n"; - TableKey table_key = group.get_table_keys()[get_next_token(s) % group.size()]; - TableRef t = group.get_table(table_key); - if (t->get_column_count() > 0 && t->size() > 0) { - auto all_col_keys = t->get_column_keys(); - size_t ndx = get_next_token(s) % all_col_keys.size(); - ColKey col = all_col_keys[ndx]; - ObjKey key = t->get_object(get_int32(s) % t->size()).get_key(); - log << "group.get_table(" << table_key << ")->get_object(" << key << ").is_null(" << col << ");\n"; - bool res = t->get_object(key).is_null(col); - static_cast(res); - } -} - -DataType FuzzObject::get_type(unsigned char c) const -{ - DataType types[] = {type_Int, type_Bool, type_Float, type_Double, type_String, type_Binary, type_Timestamp}; - - unsigned char mod = c % (sizeof(types) / sizeof(DataType)); - return types[mod]; -} - -const char* FuzzObject::get_encryption_key() const -{ -#if REALM_ENABLE_ENCRYPTION - return "1234567890123456789012345678901123456789012345678901234567890123"; -#else - return nullptr; -#endif -} - -int64_t FuzzObject::get_int64(State& s) const -{ - int64_t v = 0; - for (size_t t = 0; t < 8; t++) { - unsigned char c = get_next_token(s); - *(reinterpret_cast(&v) + t) = c; - } - return v; -} - -int32_t FuzzObject::get_int32(State& s) const -{ - int32_t v = 0; - for (size_t t = 0; t < 4; t++) { - unsigned char c = get_next_token(s); - *(reinterpret_cast(&v) + t) = c; - } - return v; -} - -std::string FuzzObject::create_string(size_t length) const -{ - REALM_ASSERT_3(length, <, 256); - static auto& chrs = "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - thread_local static std::mt19937 rg{std::random_device{}()}; - thread_local static std::uniform_int_distribution pick(0, sizeof(chrs) - 2); - std::string s; - s.reserve(length); - while (length--) - s += chrs[pick(rg)]; - return s; -} - -std::pair FuzzObject::get_timestamp_values(State& s) const -{ - int64_t seconds = get_int64(s); - int32_t nanoseconds = get_int32(s) % 1000000000; - // Make sure the values form a sensible Timestamp - const bool both_non_negative = seconds >= 0 && nanoseconds >= 0; - const bool both_non_positive = seconds <= 0 && nanoseconds <= 0; - const bool correct_timestamp = both_non_negative || both_non_positive; - if (!correct_timestamp) { - nanoseconds = -nanoseconds; - } - return {seconds, nanoseconds}; -} - -std::string FuzzObject::create_column_name(DataType t) -{ - std::string str; - switch (t) { - case type_Int: - str = "int_"; - break; - case type_Bool: - str = "bool_"; - break; - case type_Float: - str = "float_"; - break; - case type_Double: - str = "double_"; - break; - case type_String: - str = "string_"; - break; - case type_Binary: - str = "binary_"; - break; - case type_Timestamp: - str = "date_"; - break; - case type_Decimal: - str = "decimal_"; - break; - case type_ObjectId: - str = "id_"; - break; - case type_Link: - str = "link_"; - break; - case type_TypedLink: - str = "typed_link_"; - break; - case type_LinkList: - str = "link_list_"; - break; - case type_UUID: - str = "uuid_"; - break; - case type_Mixed: - str = "any_"; - break; - } - return str + util::to_string(m_column_index++); -} - -std::string FuzzObject::create_table_name() -{ - std::string str = "Table_"; - return str + util::to_string(m_table_index++); -} - -std::string FuzzObject::get_current_time_stamp() const -{ - std::time_t t = std::time(nullptr); - const int str_size = 100; - char str_buffer[str_size] = {0}; - std::strftime(str_buffer, str_size, "%c", std::localtime(&t)); - return str_buffer; -} \ No newline at end of file diff --git a/test/realm-fuzzer/fuzz_object.hpp b/test/realm-fuzzer/fuzz_object.hpp deleted file mode 100644 index a56a33bf933..00000000000 --- a/test/realm-fuzzer/fuzz_object.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ -#ifndef FUZZ_OBJECT_HPP -#define FUZZ_OBJECT_HPP - -#include "util.hpp" -#include -#include -#include -#include - -struct State; -class FuzzLog; -class FuzzObject { - // list of realm operations we support in our fuzzer -public: - void create_table(realm::Group& group, FuzzLog& log); - void remove_table(realm::Group& group, FuzzLog& lo, State& sg); - void clear_table(realm::Group& group, FuzzLog& log, State& s); - void create_object(realm::Group& group, FuzzLog& log, State& s); - void add_column(realm::Group& group, FuzzLog& log, State& s); - void remove_column(realm::Group& group, FuzzLog& log, State& s); - void rename_column(realm::Group& group, FuzzLog& log, State& s); - void add_search_index(realm::Group& group, FuzzLog& log, State& s); - void remove_search_index(realm::Group& group, FuzzLog& log, State& s); - void add_column_link(realm::Group& group, FuzzLog& log, State& s); - void add_column_link_list(realm::Group& group, FuzzLog& log, State& s); - void set_obj(realm::Group& group, FuzzLog& log, State& s); - void remove_obj(realm::Group& group, FuzzLog& log, State& s); - void remove_recursive(realm::Group& group, FuzzLog& log, State& s); - void enumerate_column(realm::Group& group, FuzzLog& log, State& s); - void get_all_column_names(realm::Group& group, FuzzLog& log); - void commit(realm::SharedRealm shared_realm, FuzzLog& log); - void rollback(realm::SharedRealm shared_realm, realm::Group& group, FuzzLog& log); - void advance(realm::SharedRealm shared_realm, FuzzLog& log); - void close_and_reopen(realm::SharedRealm& shared_realm, FuzzLog& log, const realm::Realm::Config& config); - void create_table_view(realm::Group& group, FuzzLog& log, State& s, std::vector& table_views); - void check_null(realm::Group& group, FuzzLog& log, State& s); - - const char* get_encryption_key() const; - std::string get_current_time_stamp() const; - unsigned char get_next_token(State&) const; - -private: - realm::DataType get_type(unsigned char c) const; - int64_t get_int64(State& s) const; - int32_t get_int32(State& s) const; - std::string create_string(size_t length) const; - std::pair get_timestamp_values(State& s) const; - std::string create_column_name(realm::DataType t); - std::string create_table_name(); - - int m_table_index = 0; - int m_column_index = 0; -}; -#endif \ No newline at end of file diff --git a/test/realm-fuzzer/libfuzzer_runner.cpp b/test/realm-fuzzer/libfuzzer_runner.cpp deleted file mode 100644 index 73998799607..00000000000 --- a/test/realm-fuzzer/libfuzzer_runner.cpp +++ /dev/null @@ -1,32 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ -#include "fuzz_engine.hpp" -#include - -// This function is the entry point for libfuzzer, main is auto-generated -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size); - -int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) -{ - if (Size == 0) - return 0; - std::string input{(const char*)Data, Size}; - FuzzEngine fuzz_engine; - return fuzz_engine.run_fuzzer(input, "realm_libfuzz", false, - "realm-libfuzz.txt"); // run the fuzzer with no logging -} diff --git a/test/realm-fuzzer/scripts/start_fuzz_afl.sh b/test/realm-fuzzer/scripts/start_fuzz_afl.sh deleted file mode 100755 index 5627eb2f36c..00000000000 --- a/test/realm-fuzzer/scripts/start_fuzz_afl.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT=$(basename "${BASH_SOURCE[0]}") -ROOT_DIR=$(git rev-parse --show-toplevel) -BUILD_DIR="build.realm.fuzzer.afl" - -build_mode="Debug" -num_fuzzers="1" -fuzz_test="realm-afl++" - - -if [ "$#" -ne 2 ]; then - echo "Usage: ${SCRIPT} " - echo " num_fuzzers : the number of fuzzers to run in parallel. Default ${num_fuzzers}." - echo " build mode : either Debug or Release. Default ${build_mode}." -fi - -if ! [[ -z "$1" ]]; then - num_fuzzers = "$1" -fi -if ! [[ -z "$2" ]]; then - build_mode = "$2" -fi - -if [ "$(uname)" = "Darwin" ]; then - # FIXME: Consider detecting if ReportCrash was already unloaded and skip this message - # or print and don't try to run AFL. - echo "----------------------------------------------------------------------------------------" - echo "Make sure you have unloaded the OS X crash reporter:" - echo - echo "launchctl unload -w /System/Library/LaunchAgents/com.apple.ReportCrash.plist" - echo "sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.ReportCrash.Root.plist" - echo "----------------------------------------------------------------------------------------" -else - # FIXME: Check if AFL works if the core pattern is different, but does not start with | and test for that - if [ "$(cat /proc/sys/kernel/core_pattern)" != "core" ]; then - echo "----------------------------------------------------------------------------------------" - echo "AFL might mistake crashes with hangs if the core is outputed to an external process" - echo "Please run:" - echo - echo "sudo sh -c 'echo core > /proc/sys/kernel/core_pattern'" - echo "----------------------------------------------------------------------------------------" - exit 1 - fi -fi - -echo "Building..." - -cd "${ROOT_DIR}" || exit - -mkdir -p "${BUILD_DIR}" - -cd "${BUILD_DIR}" || exit -if [ -z ${REALM_MAX_BPNODE_SIZE} ]; then - REALM_MAX_BPNODE_SIZE=$(python -c "import random; print ((random.randint(4,999), 1000)[bool(random.randint(0,1))])") -fi - -cmake -D CMAKE_BUILD_TYPE=${build_mode} \ - -D CMAKE_C_COMPILER=afl-cc \ - -D CMAKE_CXX_COMPILER=afl-c++ \ - -D REALM_MAX_BPNODE_SIZE="${REALM_MAX_BPNODE_SIZE}" \ - -D REALM_ENABLE_ENCRYPTION=OFF \ - -G Ninja \ - .. - -ninja "${fuzz_test}" - -echo "Cleaning up the findings directory" - -FINDINGS_DIR="findings" -EXEC=$(find . -name ${fuzz_test}) - -pkill afl-fuzz -rm -rf "${FINDINGS_DIR}" -mkdir -p "${FINDINGS_DIR}" - -# see also stop_parallel_fuzzer.sh -time_out="1000" # ms -memory="1000" # MB - -echo "Going to fuzz with AFL++: ${PWD}/${EXEC}" - -# if we have only one fuzzer -if [ "${num_fuzzers}" -eq 1 ]; then - afl-fuzz -t "$time_out" \ - -m "$memory" \ - -i "${ROOT_DIR}/test/realm-fuzzer/testcases" \ - -o "${FINDINGS_DIR}" \ - ${EXEC} @@ - exit 0 -fi - -# start the fuzzers in parallel -echo "Starting $num_fuzzers fuzzers in parallel" -for i in $(seq 1 ${num_fuzzers}); do - [[ $i -eq 1 ]] && flag="-M" || flag="-S" - afl-fuzz -t "$time_out" \ - -m "$memory" \ - -i "${ROOT_DIR}/test/realm-fuzzer/testcases" \ - -o "${FINDINGS_DIR}" \ - "${flag}" "fuzzer$i" \ - ${EXEC} @@ --name "fuzzer$i" >/dev/null 2>&1 & -done - -echo -echo "Use 'afl-whatsup ${ROOT_DIR}/${BUILD_DIR}/${FINDINGS_DIR}' to check progress" -echo \ No newline at end of file diff --git a/test/realm-fuzzer/scripts/start_lib_fuzzer.sh b/test/realm-fuzzer/scripts/start_lib_fuzzer.sh deleted file mode 100755 index 15fe3c68cbb..00000000000 --- a/test/realm-fuzzer/scripts/start_lib_fuzzer.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT=$(basename "${BASH_SOURCE[0]}") -ROOT_DIR=$(git rev-parse --show-toplevel) -BUILD_DIR="build.realm.fuzzer.libfuzz" - -build_mode="Debug" -corpus="" -fuzz_test="realm-libfuzz" - -if [ "$#" -ne 2 ]; then - echo "Usage: ${SCRIPT} " - echo "build mode : either Debug or Release. Default ${build_mode}." - echo "corpus (initial path seed for fuzzing) : e.g ./test/test.txt. Default no seed used." -fi - -if ! [[ -z "$1" ]]; then - build_mode="$1" -fi -if ! [[ -z "$2" ]]; then - corpus="$2" -fi - -echo "Building..." - -cd "${ROOT_DIR}" || exit - -mkdir -p "${BUILD_DIR}" - -cd "${BUILD_DIR}" || exit -if [ -z ${REALM_MAX_BPNODE_SIZE} ]; then - REALM_MAX_BPNODE_SIZE=$(python -c "import random; print ((random.randint(4,999), 1000)[bool(random.randint(0,1))])") -fi - -cmake -D REALM_LIBFUZZER=ON \ - -D CMAKE_BUILD_TYPE=${build_mode} \ - -D CMAKE_C_COMPILER=clang \ - -D CMAKE_CXX_COMPILER=clang++ \ - -D REALM_MAX_BPNODE_SIZE="${REALM_MAX_BPNODE_SIZE}" \ - -D REALM_ENABLE_ENCRYPTION=OFF \ - -G Ninja \ - .. - -ninja "${fuzz_test}" -EXEC=$(find . -name ${fuzz_test}) -echo "Going to fuzz with LibFuzz: ${PWD}/${EXEC}" -./${EXEC} ${corpus} \ No newline at end of file diff --git a/test/realm-fuzzer/util.hpp b/test/realm-fuzzer/util.hpp deleted file mode 100644 index 8ec2cce7a14..00000000000 --- a/test/realm-fuzzer/util.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#ifndef FUZZ_UTIL_HPP -#define FUZZ_UTIL_HPP - -#include - -struct State { - std::string str; - size_t pos; -}; - -struct EndOfFile { -}; - -enum Instruction { - Add_Table = 0, - Remove_Table = 1, - Create_Object = 2, - Rename_Column = 3, - Add_Column = 4, - Remove_Column = 5, - Set = 6, - Remove_Object = 7, - Remove_Recursive = 8, - Add_Column_Link = 9, - Add_Column_Link_List = 10, - Clear_Table = 11, - Add_Search_Index = 12, - Remove_Search_Index = 13, - Commit = 14, - Rollback = 15, - Advance = 16, - Move_Last_Over = 17, - Close_And_Reopen = 18, - Get_All_Column_Names = 19, - Create_Table_View = 20, - Compact = 21, - Is_Null = 22, - Enumerate_Column = 23, - Count = 24 -}; - - -#define TEST_FUZZ -// #ifdef TEST_FUZZ -// Determines whether or not to run the shared group verify function -// after each transaction. This will find errors earlier but is expensive. -#define REALM_VERIFY true - -#if REALM_VERIFY -#define REALM_DO_IF_VERIFY(log, op) \ - do { \ - log << #op << ";\n"; \ - op; \ - } while (false) -#else -#define REALM_DO_IF_VERIFY(log, owner) \ - do { \ - } while (false) -#endif - -#endif \ No newline at end of file diff --git a/test/sync_fixtures.hpp b/test/sync_fixtures.hpp index ca1f656439f..4f1c231dee3 100644 --- a/test/sync_fixtures.hpp +++ b/test/sync_fixtures.hpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -302,11 +301,10 @@ class HTTPRequestClient { util::PrefixLogger logger; - HTTPRequestClient(const std::shared_ptr& logger_ptr, const network::Endpoint& endpoint, - const HTTPRequest& request) - : logger{"HTTP client: ", logger_ptr} + HTTPRequestClient(util::Logger& logger, const network::Endpoint& endpoint, const HTTPRequest& request) + : logger{"HTTP client: ", logger} , m_endpoint{endpoint} - , m_http_client{*this, logger_ptr} + , m_http_client{*this, logger} , m_request{request} { } @@ -535,10 +533,7 @@ class MultiClientServerFixture { m_clients.resize(num_clients); for (int i = 0; i < num_clients; ++i) { Client::Config config_2; - - m_client_socket_providers.push_back(std::make_shared( - m_client_loggers[i], "", websocket::DefaultSocketProvider::AutoStart{false})); - config_2.socket_provider = m_client_socket_providers.back(); + config_2.user_agent_application_info = "TestFixture/" REALM_VERSION_STRING; config_2.logger = m_client_loggers[i]; config_2.reconnect_mode = ReconnectMode::testing; config_2.ping_keepalive_period = config.client_ping_period; @@ -551,6 +546,7 @@ class MultiClientServerFixture { } m_server_threads.resize(num_servers); + m_client_threads.resize(num_clients); m_simulated_server_error_rates.resize(num_servers); m_simulated_client_error_rates.resize(num_clients); @@ -566,8 +562,10 @@ class MultiClientServerFixture { { unit_test::TestContext& test_context = m_test_context; stop(); - m_clients.clear(); - m_client_socket_providers.clear(); + for (int i = 0; i < m_num_clients; ++i) { + if (m_client_threads[i].joinable()) + CHECK(!m_client_threads[i].join()); + } for (int i = 0; i < m_num_servers; ++i) { if (m_server_threads[i].joinable()) CHECK(!m_server_threads[i].join()); @@ -595,26 +593,15 @@ class MultiClientServerFixture { m_connection_state_change_listeners[client_index] = std::move(handler_2); } + // Must be called before start(). void set_client_side_error_rate(int client_index, int n, int m) { - REALM_ASSERT(client_index >= 0 && client_index < m_num_clients); - auto sim = std::make_pair(n, m); - // Save the simulated error rate - m_simulated_client_error_rates[client_index] = sim; - - // Post the new simulated error rate - using sf = _impl::SimulatedFailure; - // Post it onto the event loop to update the event loop thread - m_client_socket_providers[client_index]->post([sim = std::move(sim)](Status) { - sf::prime_random(sf::sync_client__read_head, sim.first, sim.second, - random_int()); // Seed from global generator - }); + m_simulated_client_error_rates[client_index] = std::make_pair(n, m); } // Must be called before start(). void set_server_side_error_rate(int server_index, int n, int m) { - REALM_ASSERT(server_index >= 0 && server_index < m_num_servers); m_simulated_server_error_rates[server_index] = std::make_pair(n, m); } @@ -624,16 +611,10 @@ class MultiClientServerFixture { m_server_threads[i].start([this, i] { run_server(i); }); - - for (int i = 0; i < m_num_clients; ++i) { - m_client_socket_providers[i]->start(); - } - } - - void start_client(int index) - { - REALM_ASSERT(index >= 0 && index < m_num_clients); - m_client_socket_providers[index]->start(); + for (int i = 0; i < m_num_clients; ++i) + m_client_threads[i].start([this, i] { + run_client(i); + }); } // Use either the methods below or `start()`. @@ -645,6 +626,14 @@ class MultiClientServerFixture { }); } + void start_client(int index) + { + REALM_ASSERT(index >= 0 && index < m_num_clients); + m_client_threads[index].start([this, index] { + run_client(index); + }); + } + void stop_server(int index) { REALM_ASSERT(index >= 0 && index < m_num_servers); @@ -658,17 +647,10 @@ class MultiClientServerFixture { void stop_client(int index) { REALM_ASSERT(index >= 0 && index < m_num_clients); - auto& client = get_client(index); - auto sim = m_simulated_client_error_rates[index]; - if (sim.first != 0) { - using sf = _impl::SimulatedFailure; - // If we're using a simulated failure, clear it by posting onto the event loop - m_client_socket_providers[index]->post([](Status) mutable { - sf::unprime(sf::sync_client__read_head); // Clear the sim failure set when started - }); - } - // We can't wait for clearing the simulated failure since some tests stop the client early - client.drain(); + m_clients[index]->stop(); + unit_test::TestContext& test_context = m_test_context; + if (m_client_threads[index].joinable()) + CHECK(!m_client_threads[index].join()); } void stop() @@ -813,7 +795,7 @@ class MultiClientServerFixture { std::vector> m_connection_state_change_listeners; std::vector m_server_ports; std::vector m_server_threads; - std::vector> m_client_socket_providers; + std::vector m_client_threads; std::vector> m_simulated_server_error_rates; std::vector> m_simulated_client_error_rates; std::vector m_allow_server_errors; @@ -852,6 +834,28 @@ class MultiClientServerFixture { stop(); m_server_loggers[i]->error("Exception was throw from server[%1]'s event loop", i + 1); } + + void run_client(int i) + { + auto do_run_client = [this, i] { + auto sim = m_simulated_client_error_rates[i]; + if (sim.first != 0) { + using sf = _impl::SimulatedFailure; + sf::RandomPrimeGuard pg(sf::sync_client__read_head, sim.first, sim.second, + random_int()); // Seed from global generator + m_clients[i]->run(); + } + else { + m_clients[i]->run(); + } + m_clients[i]->stop(); + }; + unit_test::TestContext& test_context = m_test_context; + if (CHECK_NOTHROW(do_run_client())) + return; + stop(); + m_server_loggers[i]->error("Exception was throw from client[%1]'s event loop", i + 1); + } }; diff --git a/test/test_client_reset.cpp b/test/test_client_reset.cpp index dfcdcf035f1..525bb78b9c6 100644 --- a/test/test_client_reset.cpp +++ b/test/test_client_reset.cpp @@ -6,9 +6,6 @@ #include #include -#include -#include - #include "test.hpp" #include "sync_fixtures.hpp" #include "util/semaphore.hpp" @@ -23,47 +20,6 @@ namespace { using ErrorInfo = Session::ErrorInfo; -TEST(ClientReset_TransferGroupWithDanglingLinks) -{ - SHARED_GROUP_TEST_PATH(path_1); - SHARED_GROUP_TEST_PATH(path_2); - - auto setup_realm = [](auto& path) { - DBRef sg = DB::create(make_client_replication(), path); - - auto wt = sg->start_write(); - - // The ordering of creating the tables matters here. The bug this test is verifying depends - // on tablekeys being created such that the table that links come from is transferred before - // the table that links are linking to. - auto table = wt->add_table_with_primary_key("class_table", type_String, "_id"); - auto target = wt->add_table_with_primary_key("class_target", type_Int, "_id"); - table->add_column_list(*target, "list"); - auto obj = table->create_object_with_primary_key(Mixed{"the_object"}); - auto lst = obj.get_linklist("list"); - for (int64_t i = 0; i < 10; ++i) { - target->create_object_with_primary_key(i); - lst.add(target->create_object_with_primary_key(i).get_key()); - } - wt->commit(); - - return sg; - }; - - auto sg_1 = setup_realm(path_1); - auto sg_2 = setup_realm(path_2); - - auto rt = sg_1->start_read(); - auto wt = sg_2->start_write(); - - auto target_2 = wt->get_table("class_target"); - auto obj = target_2->get_object_with_primary_key(Mixed{5}); - obj.invalidate(); - - wt->commit_and_continue_writing(); - _impl::client_reset::transfer_group(*rt, *wt, *test_context.logger); -} - TEST(ClientReset_NoLocalChanges) { TEST_DIR(dir_1); // The original server dir. diff --git a/test/test_file.cpp b/test/test_file.cpp index 3c86824b58d..ed0b362153c 100644 --- a/test/test_file.cpp +++ b/test/test_file.cpp @@ -531,6 +531,7 @@ TEST(File_parent_dir) } } +#ifndef _WIN32 TEST(File_GetUniqueID) { TEST_PATH(path_1); @@ -551,15 +552,16 @@ TEST(File_GetUniqueID) File::UniqueID uid1_1 = file1_1.get_unique_id(); File::UniqueID uid1_2 = file1_2.get_unique_id(); File::UniqueID uid2_1 = file2_1.get_unique_id(); - std::optional uid2_2; - CHECK(uid2_2 = File::get_unique_id(path_2)); + File::UniqueID uid2_2; + CHECK(File::get_unique_id(path_2, uid2_2)); CHECK(uid1_1 == uid1_2); - CHECK(uid2_1 == *uid2_2); + CHECK(uid2_1 == uid2_2); CHECK(uid1_1 != uid2_1); // Path doesn't exist - CHECK_NOT(File::get_unique_id(path_3)); + File::UniqueID uid3_1; + CHECK_NOT(File::get_unique_id(path_3, uid3_1)); // Test operator< File::UniqueID uid4_1{0, 5}; @@ -576,5 +578,6 @@ TEST(File_GetUniqueID) CHECK_NOT(uid4_1 < uid4_2); CHECK_NOT(uid4_2 < uid4_1); } +#endif #endif // TEST_FILE diff --git a/test/test_file_locks.cpp b/test/test_file_locks.cpp index 96412d269b0..bb4aa87d3cb 100644 --- a/test/test_file_locks.cpp +++ b/test/test_file_locks.cpp @@ -128,9 +128,9 @@ TEST(File_NoSpuriousTryLockFailures) try { File file(path, File::mode_Write); for (int i = 0; i != num_rounds; ++i) { - bool good_lock = file.try_rw_lock_exclusive(); + bool good_lock = file.try_lock_exclusive(); if (good_lock) - file.rw_unlock(); + file.unlock(); { LockGuard l(mutex); if (good_lock) @@ -208,7 +208,7 @@ TEST_IF(File_NoSpuriousTryLockFailures2, !(running_with_valgrind || running_with } // All threads race for the lock - bool owns_lock = file.try_rw_lock_exclusive(); + bool owns_lock = file.try_lock_exclusive(); barrier_2 = 0; @@ -226,7 +226,7 @@ TEST_IF(File_NoSpuriousTryLockFailures2, !(running_with_valgrind || running_with CHECK_EQUAL(lock_taken.load(), size_t(1)); if(owns_lock) { - file.rw_unlock(); + file.unlock(); } barrier_1 = 0; diff --git a/test/test_handshake.cpp b/test/test_handshake.cpp new file mode 100644 index 00000000000..f2d565b4201 --- /dev/null +++ b/test/test_handshake.cpp @@ -0,0 +1,523 @@ +#include +#include +#include +#include + +#include "test.hpp" +#include "util/thread_wrapper.hpp" + +using namespace realm; +using namespace realm::sync; +using namespace realm::test_util; + +using port_type = network::Endpoint::port_type; +using ConnectionStateChangeListener = Session::ConnectionStateChangeListener; +using ErrorInfo = Session::ErrorInfo; + +namespace { + +// SurpriseServer is a server that listens on a port accepts a single +// connection waits for a HTTP request and returns a HTTP response. +// The response depends on the URL of the request. For instance, +// a request to /realm-sync/301 will send a +// HTTP/1.1 301 Moved Permanently response. +class SurpriseServer { +public: + SurpriseServer(util::Logger& logger) + : m_acceptor{m_service} + , m_socket{m_service} + , m_http_server{*this, logger} + { + } + + void start() + { + m_acceptor.open(network::StreamProtocol::ip_v4()); + m_acceptor.listen(); + + auto handler = [this](std::error_code ec) { + REALM_ASSERT(!ec); + this->handle_accept(); // Throws + }; + m_acceptor.async_accept(m_socket, handler); // Throws + } + + void run() + { + m_service.run(); + } + + void stop() + { + m_service.stop(); + } + + network::Endpoint listen_endpoint() const + { + return m_acceptor.local_endpoint(); + } + + void async_read_until(char* buffer, size_t size, char delim, std::function handler) + { + m_socket.async_read_until(buffer, size, delim, m_read_ahead_buffer, handler); // Throws + } + + void async_read(char* buffer, size_t size, std::function handler) + { + m_socket.async_read(buffer, size, m_read_ahead_buffer, handler); // Throws + } + +private: + network::Service m_service; + network::Acceptor m_acceptor; + network::Socket m_socket; + network::ReadAheadBuffer m_read_ahead_buffer; + HTTPServer m_http_server; + std::string m_response; + + void handle_accept() + { + auto handler = [this](HTTPRequest request, std::error_code ec) { + REALM_ASSERT(!ec); + this->handle_http_request(request); // Throws + }; + m_http_server.async_receive_request(std::move(handler)); // Throws + } + + void handle_http_request(const HTTPRequest& request) + { + const std::string& path = request.path; + const std::string expected_prefix = "/realm-sync/%2F"; + REALM_ASSERT(path.compare(0, expected_prefix.size(), expected_prefix) == 0); + std::string key = path.substr(expected_prefix.size()); + if (key == "http_1_0") + send_http_1_0(); + else if (key == "invalid-status-code") + send_invalid_status_code(); + else if (key == "missing-websocket-headers") + send_missing_websocket_headers(); + else if (key == "200") + send_200(); + else if (key == "201") + send_201(); + else if (key == "300") + send_300(); + else if (key == "301") + send_301(); + else if (key == "400") + send_400(); + else if (key == "401") + send_401(); + else if (key == "403") + send_403(); + else if (key == "404") + send_404(); + else if (key == "500") + send_500(); + else if (key == "501") + send_501(); + else if (key == "502") + send_502(); + else if (key == "503") + send_503(); + else if (key == "504") + send_504(); + else + send_nothing(); + } + + void send_response() + { + auto handler = [=](std::error_code ec, size_t nwritten) { + REALM_ASSERT(!ec); + REALM_ASSERT(nwritten == m_response.size()); + }; + m_socket.async_write(m_response.data(), m_response.size(), handler); + } + + void send_http_1_0() + { + m_response = "HTTP/1.0 200 OK\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_invalid_status_code() + { + m_response = "HTTP/1.1 99999 Strange\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_missing_websocket_headers() + { + m_response = "HTTP/1.1 101 Switching Protocols\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_200() + { + m_response = "HTTP/1.1 200 OK\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_201() + { + m_response = "HTTP/1.1 201 Created\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_300() + { + m_response = "HTTP/1.1 300 Multiple Choices\r\n" + "Server: surprise-server\r\n" + "Location: http://10.0.0.0\r\n" + "\r\n"; + send_response(); + } + + void send_301() + { + m_response = "HTTP/1.1 301 Moved Permanently\r\n" + "Server: surprise-server\r\n" + "Location: http://10.0.0.0\r\n" + "\r\n"; + send_response(); + } + + void send_400() + { + m_response = "HTTP/1.1 400 Bad Request\r\n" + "Server: surprise-server\r\n" + "Location: http://10.0.0.0\r\n" + "\r\n"; + send_response(); + } + + void send_401() + { + m_response = "HTTP/1.1 401 Unauthorized\r\n" + "Server: surprise-server\r\n" + "Location: http://10.0.0.0\r\n" + "\r\n"; + send_response(); + } + + void send_403() + { + m_response = "HTTP/1.1 403 Forbidden\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_404() + { + m_response = "HTTP/1.1 404 Not Found\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_500() + { + m_response = "HTTP/1.1 500 Internal Server Error\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_501() + { + m_response = "HTTP/1.1 501 Not Implemented\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_502() + { + m_response = "HTTP/1.1 502 Bad Gateway\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_503() + { + m_response = "HTTP/1.1 503 Service Unavailable\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_504() + { + m_response = "HTTP/1.1 504 Gateway Timeout\r\n" + "Server: surprise-server\r\n" + "\r\n"; + send_response(); + } + + void send_nothing() + { + // no-op + } +}; + +// This function creates a Surprise Server and a sync client, lets the sync +// client initiate a sync connection which the surprise server responds to. +// The response depends on the server path. The check is that the clients +// ConnectionStateChangeListener is called with the proper error code and +// is_fatal value. +void run_client_surprise_server(unit_test::TestContext& test_context, const std::string server_path, + std::error_code ec, bool is_fatal) +{ + SHARED_GROUP_TEST_PATH(path); + + util::Logger& logger = test_context.logger; + util::PrefixLogger server_logger("Server: ", logger); + util::PrefixLogger client_logger("Client: ", logger); + + SurpriseServer server{server_logger}; + server.start(); + ThreadWrapper server_thread; + server_thread.start([&] { + server.run(); + }); + + Client::Config client_config; + client_config.logger = &client_logger; + client_config.one_connection_per_session = true; + client_config.tcp_no_delay = true; + Client client(client_config); + + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + + Session::Config session_config; + session_config.server_address = "localhost"; + session_config.server_port = server.listen_endpoint().port(); + session_config.server_path = server_path; + + Session session{client, path, session_config}; + + std::function connection_state_listener = [&](ConnectionState connection_state, + const ErrorInfo* error_info) { + if (error_info) { + CHECK(connection_state == ConnectionState::disconnected); + CHECK_EQUAL(ec, error_info->error_code); + CHECK_EQUAL(is_fatal, error_info->is_fatal); + client.stop(); + } + }; + session.set_connection_state_change_listener(connection_state_listener); + session.bind(); + session.wait_for_download_complete_or_client_stopped(); + + client.stop(); + client_thread.join(); + server.stop(); + server_thread.join(); +} + +} // unnamed namespace + + +namespace { + +TEST(Handshake_HTTP_Version) +{ + const std::string server_path = "/http_1_0"; + std::error_code ec = websocket::Error::bad_response_invalid_http; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_InvalidStatusCode) +{ + const std::string server_path = "/invalid-status-code"; + std::error_code ec = websocket::Error::bad_response_invalid_http; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_MissingWebSocketHeaders) +{ + const std::string server_path = "/missing-websocket-headers"; + std::error_code ec = websocket::Error::bad_response_header_protocol_violation; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_200) +{ + const std::string server_path = "/200"; + std::error_code ec = websocket::Error::bad_response_200_ok; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_201) +{ + const std::string server_path = "/201"; + std::error_code ec = websocket::Error::bad_response_2xx_successful; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_300) +{ + const std::string server_path = "/300"; + std::error_code ec = websocket::Error::bad_response_3xx_redirection; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_301) +{ + const std::string server_path = "/301"; + std::error_code ec = websocket::Error::bad_response_301_moved_permanently; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_400) +{ + const std::string server_path = "/400"; + std::error_code ec = websocket::Error::bad_response_4xx_client_errors; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_401) +{ + const std::string server_path = "/401"; + std::error_code ec = websocket::Error::bad_response_401_unauthorized; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_403) +{ + const std::string server_path = "/403"; + std::error_code ec = websocket::Error::bad_response_403_forbidden; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_404) +{ + const std::string server_path = "/404"; + std::error_code ec = websocket::Error::bad_response_404_not_found; + bool is_fatal = true; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_500) +{ + const std::string server_path = "/500"; + std::error_code ec = websocket::Error::bad_response_500_internal_server_error; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_501) +{ + const std::string server_path = "/501"; + std::error_code ec = websocket::Error::bad_response_5xx_server_error; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_502) +{ + const std::string server_path = "/502"; + std::error_code ec = websocket::Error::bad_response_502_bad_gateway; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_503) +{ + const std::string server_path = "/503"; + std::error_code ec = websocket::Error::bad_response_503_service_unavailable; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +TEST(Handshake_504) +{ + const std::string server_path = "/504"; + std::error_code ec = websocket::Error::bad_response_504_gateway_timeout; + bool is_fatal = false; + run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +// Enable when the client gets a handshake timeout. +TEST_IF(Handshake_Timeout, false) +{ + // const std::string server_path = "/nothing"; + // std::error_code ec = websocket::Error::; // CHANGE + // bool is_fatal = false; + // run_client_surprise_server(test_context, server_path, ec, is_fatal); +} + +// Test connection to external server. This test should Only be enabled +// during testing. +TEST_IF(Handshake_ExternalServer, false) +{ + const std::string server_address = "www.realm.io"; + port_type server_port = 80; + + SHARED_GROUP_TEST_PATH(path); + util::Logger& logger = test_context.logger; + util::PrefixLogger client_logger("Client: ", logger); + + Client::Config client_config; + client_config.logger = &client_logger; + client_config.one_connection_per_session = true; + client_config.tcp_no_delay = true; + Client client(client_config); + + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + + Session::Config session_config; + session_config.server_address = server_address; + session_config.server_port = server_port; + session_config.server_path = "/default"; + + Session session{client, path, session_config}; + + std::function connection_state_listener = [&](ConnectionState connection_state, + const ErrorInfo* error_info) { + if (error_info) { + CHECK(connection_state == ConnectionState::disconnected); + std::error_code ec = websocket::Error::bad_response_301_moved_permanently; + CHECK_EQUAL(ec, error_info->error_code); + CHECK_EQUAL(true, error_info->is_fatal); + client.stop(); + } + }; + session.set_connection_state_change_listener(connection_state_listener); + session.bind(); + session.wait_for_download_complete_or_client_stopped(); + + client.stop(); + client_thread.join(); +} + +} // unnamed namespace diff --git a/test/test_metrics.cpp b/test/test_metrics.cpp index 1f88f985620..b0d8b119e04 100644 --- a/test/test_metrics.cpp +++ b/test/test_metrics.cpp @@ -796,14 +796,7 @@ NONCONCURRENT_TEST(Metrics_TransactionTimings) metrics::TransactionInfo::TransactionType::write_transaction); CHECK_GREATER(transactions->at(3).get_transaction_time_nanoseconds(), 10'000); // > 10us CHECK_LESS(transactions->at(3).get_transaction_time_nanoseconds(), 2'000'000'000); // < 2s - // TODO: Investigate why this check is failing, since it sometimes fails and, since this is a - // non-concurrent test, the sync_to_disk setting should not change during the execution of this test. -#if 0 - if (!get_disable_sync_to_disk()) { - // This check returns 0 for get_fsync_time_nanoseconds if sync to disk is disabled - CHECK_GREATER(transactions->at(3).get_fsync_time_nanoseconds(), 0); // fsync on write takes some time - } -#endif + CHECK_GREATER(transactions->at(3).get_fsync_time_nanoseconds(), 0); // fsync on write takes some time } diff --git a/test/test_mixed_null_assertions.cpp b/test/test_mixed_null_assertions.cpp index 33a5ea2984a..1511f819256 100644 --- a/test/test_mixed_null_assertions.cpp +++ b/test/test_mixed_null_assertions.cpp @@ -63,13 +63,6 @@ TEST(List_Mixed_do_set) set.insert_null(0); set.set(0, Mixed("hello world")); - auto val = set.get(0); - CHECK(val.is_type(type_String)); - CHECK_EQUAL(val.get_string(), "hello world"); - set.set(0, Mixed(BinaryData("hello world", 11))); - val = set.get(0); - CHECK(val.is_type(type_Binary)); - CHECK_EQUAL(val.get_binary(), BinaryData("hello world", 11)); } TEST(List_Mixed_do_insert) diff --git a/test/test_set.cpp b/test/test_set.cpp index e836d81d03a..709cb16dcd3 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -158,55 +158,6 @@ TEST(Set_Mixed) CHECK(ref_values == actuals); } -TEST(Set_Mixed_SortStringAndBinary) -{ - Group g; - auto table = g.add_table("table"); - auto col = table->add_column_set(type_Mixed, "set"); - auto obj = table->create_object(); - auto set = obj.get_set(col); - - std::vector indices = {1}; - - // Empty set - set.sort(indices); - CHECK(indices.empty()); - - // Strings only - set.insert("c"); - set.insert("e"); - set.insert("a"); - set.sort(indices, true); - CHECK_EQUAL(indices, (std::vector{0, 1, 2})); - set.sort(indices, false); - CHECK_EQUAL(indices, (std::vector{2, 1, 0})); - - // Non-strings surrounding the strings - set.insert(0); - set.insert(UUID()); - set.sort(indices, true); - CHECK_EQUAL(indices, (std::vector{0, 1, 2, 3, 4})); - set.sort(indices, false); - CHECK_EQUAL(indices, (std::vector{4, 3, 2, 1, 0})); - - // Binary values which should be interleaved with the strings - set.insert(BinaryData("b", 1)); - set.insert(BinaryData("d", 1)); - set.sort(indices, true); - CHECK_EQUAL(indices, (std::vector{0, 1, 4, 2, 5, 3, 6})); - set.sort(indices, false); - CHECK_EQUAL(indices, (std::vector{6, 3, 5, 2, 4, 1, 0})); - - // Non-empty but no strings - set.clear(); - set.insert(1); - set.insert(2); - set.sort(indices, true); - CHECK_EQUAL(indices, (std::vector{0, 1})); - set.sort(indices, false); - CHECK_EQUAL(indices, (std::vector{1, 0})); -} - TEST(Set_LinksRemoveBacklinks) { SHARED_GROUP_TEST_PATH(path); diff --git a/test/test_shared.cpp b/test/test_shared.cpp index d322585108a..2ef908cd154 100644 --- a/test/test_shared.cpp +++ b/test/test_shared.cpp @@ -2960,7 +2960,7 @@ TEST(Shared_LockFileInitSpinsOnZeroSize) Thread t; auto do_async = [&]() { File f(path.get_lock_path(), File::mode_Write); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); @@ -3007,7 +3007,7 @@ TEST(Shared_LockFileSpinsOnInitComplete) Thread t; auto do_async = [&]() { File f(path.get_lock_path(), File::mode_Write); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); @@ -3059,7 +3059,7 @@ TEST(Shared_LockFileOfWrongSizeThrows) auto do_async = [&]() { File f(path.get_lock_path(), File::mode_Write); f.set_fifo_path(std::string(path) + ".management", "lock.fifo"); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); @@ -3125,7 +3125,7 @@ TEST(Shared_LockFileOfWrongVersionThrows) f.open(path.get_lock_path(), File::access_ReadWrite, File::create_Auto, 0); // Throws f.set_fifo_path(std::string(path) + ".management", "lock.fifo"); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); @@ -3177,7 +3177,7 @@ TEST(Shared_LockFileOfWrongMutexSizeThrows) File f; f.open(path.get_lock_path(), File::access_ReadWrite, File::create_Auto, 0); // Throws f.set_fifo_path(std::string(path) + ".management", "lock.fifo"); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); @@ -3231,7 +3231,7 @@ TEST(Shared_LockFileOfWrongCondvarSizeThrows) File f; f.open(path.get_lock_path(), File::access_ReadWrite, File::create_Auto, 0); // Throws f.set_fifo_path(std::string(path) + ".management", "lock.fifo"); - f.rw_lock_shared(); + f.lock_shared(); File::UnlockGuard ug(f); CHECK(f.is_attached()); diff --git a/test/test_sync.cpp b/test/test_sync.cpp index fa2149fdfc3..e19aadfa838 100644 --- a/test/test_sync.cpp +++ b/test/test_sync.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -371,6 +370,7 @@ TEST(Sync_AsyncWaitCancellation) session.async_wait_for_sync_completion(sync_completion_handler); // Destruction of session cancels wait operations } + fixture.start(); bowl.get_stone(); bowl.get_stone(); @@ -1686,7 +1686,7 @@ TEST(Sync_HTTP404NotFound) HTTPRequest request; request.path = "/not-found"; - HTTPRequestClient client(test_context.logger, endpoint, request); + HTTPRequestClient client(*(test_context.logger), endpoint, request); client.fetch_response(); server.stop(); @@ -2968,11 +2968,14 @@ TEST_IF(Sync_SSL_Certificate_Verify_Callback_External, false) Client::Config config; config.logger = std::make_shared("Client: ", test_context.logger); - auto socket_provider = std::make_shared(config.logger, ""); - config.socket_provider = socket_provider; config.reconnect_mode = ReconnectMode::testing; Client client(config); + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + auto ssl_verify_callback = [&](const std::string server_address, Session::port_type server_port, const char* pem_data, size_t pem_size, int preverify_ok, int depth) { StringData pem{pem_data, pem_size}; @@ -2996,7 +2999,8 @@ TEST_IF(Sync_SSL_Certificate_Verify_Callback_External, false) session.bind(); session.wait_for_download_complete_or_client_stopped(); - client.drain(); + client.stop(); + client_thread.join(); } #endif // REALM_HAVE_OPENSSL @@ -3119,11 +3123,14 @@ TEST(Sync_UploadDownloadProgress_1) Client::Config config; config.logger = std::make_shared("Client: ", test_context.logger); - auto socket_provider = std::make_shared(config.logger, ""); - config.socket_provider = socket_provider; config.reconnect_mode = ReconnectMode::testing; Client client(config); + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + Session session(client, db, nullptr); int number_of_handler_calls = 0; @@ -3155,6 +3162,8 @@ TEST(Sync_UploadDownloadProgress_1) }); client.stop(); + client_thread.join(); + CHECK_EQUAL(number_of_handler_calls, 1); } } @@ -3383,14 +3392,16 @@ TEST(Sync_UploadDownloadProgress_3) wt.commit(); } - Client::Config client_config; client_config.logger = std::make_shared("Client: ", test_context.logger); - auto socket_provider = std::make_shared(client_config.logger, ""); - client_config.socket_provider = socket_provider; client_config.reconnect_mode = ReconnectMode::testing; Client client(client_config); + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + // when connecting to the C++ server, use URL prefix: Session::Config config; config.service_identifier = "/realm-sync"; @@ -3400,9 +3411,12 @@ TEST(Sync_UploadDownloadProgress_3) // entry is used to count the number of calls to // progress_handler. At the first call, the server is // not running, and it is started by progress_handler(). + int entry = 0; bool should_signal_cond_var = false; - auto signal_pf = util::make_promise_future(); + bool cond_var_signaled = false; + std::mutex mutex; + std::condition_variable cond_var; uint_fast64_t downloaded_bytes_1 = 123; // Not zero uint_fast64_t downloadable_bytes_1 = 123; @@ -3411,10 +3425,9 @@ TEST(Sync_UploadDownloadProgress_3) uint_fast64_t progress_version_1 = 123; uint_fast64_t snapshot_version_1 = 0; - auto progress_handler = [&, entry = int(0), promise = util::CopyablePromiseHolder(std::move(signal_pf.promise))]( - uint_fast64_t downloaded_bytes, uint_fast64_t downloadable_bytes, + auto progress_handler = [&](uint_fast64_t downloaded_bytes, uint_fast64_t downloadable_bytes, uint_fast64_t uploaded_bytes, uint_fast64_t uploadable_bytes, - uint_fast64_t progress_version, uint_fast64_t snapshot_version) mutable { + uint_fast64_t progress_version, uint_fast64_t snapshot_version) { downloaded_bytes_1 = downloaded_bytes; downloadable_bytes_1 = downloadable_bytes; uploaded_bytes_1 = uploaded_bytes; @@ -3437,7 +3450,10 @@ TEST(Sync_UploadDownloadProgress_3) } if (should_signal_cond_var) { - promise.get_promise().emplace_value(); + std::unique_lock lock(mutex); + cond_var_signaled = true; + lock.unlock(); + cond_var.notify_one(); } entry++; @@ -3474,7 +3490,12 @@ TEST(Sync_UploadDownloadProgress_3) session.nonsync_transact_notify(commited_version); } - signal_pf.future.get(); + { + std::unique_lock lock(mutex); + cond_var.wait(lock, [&] { + return cond_var_signaled; + }); + } CHECK_EQUAL(downloaded_bytes_1, 0); CHECK_EQUAL(downloadable_bytes_1, 0); @@ -3482,7 +3503,10 @@ TEST(Sync_UploadDownloadProgress_3) CHECK_NOT_EQUAL(uploadable_bytes_1, 0); CHECK_EQUAL(snapshot_version_1, commited_version); + client.stop(); + server_thread.join(); + client_thread.join(); } @@ -3606,17 +3630,18 @@ TEST(Sync_UploadDownloadProgress_5) TEST_DIR(server_dir); TEST_CLIENT_DB(db); - auto [progress_handled_promise, progress_handled] = util::make_promise_future(); + bool cond_var_signaled = false; + std::mutex mutex; + std::condition_variable cond_var; ClientServerFixture fixture(server_dir, test_context); fixture.start(); Session session = fixture.make_session(db); - auto progress_handler = [&, promise = util::CopyablePromiseHolder(std::move(progress_handled_promise))]( - uint_fast64_t downloaded_bytes, uint_fast64_t downloadable_bytes, + auto progress_handler = [&](uint_fast64_t downloaded_bytes, uint_fast64_t downloadable_bytes, uint_fast64_t uploaded_bytes, uint_fast64_t uploadable_bytes, - uint_fast64_t progress_version, uint_fast64_t snapshot_version) mutable { + uint_fast64_t progress_version, uint_fast64_t snapshot_version) { CHECK_EQUAL(downloaded_bytes, 0); CHECK_EQUAL(downloadable_bytes, 0); CHECK_EQUAL(uploaded_bytes, 0); @@ -3624,14 +3649,20 @@ TEST(Sync_UploadDownloadProgress_5) if (progress_version > 0) { CHECK_EQUAL(snapshot_version, 3); - promise.get_promise().emplace_value(); + std::unique_lock lock(mutex); + cond_var_signaled = true; + lock.unlock(); + cond_var.notify_one(); } }; session.set_progress_handler(progress_handler); + std::unique_lock lock(mutex); fixture.bind_session(session, "/test"); - progress_handled.get(); + cond_var.wait(lock, [&] { + return cond_var_signaled; + }); // The check is that we reach this point. } @@ -3663,20 +3694,24 @@ TEST(Sync_UploadDownloadProgress_6) Client::Config client_config; client_config.logger = std::make_shared("Client: ", test_context.logger); - auto socket_provider = std::make_shared(client_config.logger, ""); - client_config.socket_provider = socket_provider; client_config.reconnect_mode = ReconnectMode::testing; client_config.one_connection_per_session = false; Client client(client_config); + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + Session::Config session_config; session_config.server_address = "localhost"; session_config.server_port = server_port; session_config.realm_identifier = "/test"; session_config.signed_user_token = g_signed_test_user_token; - std::mutex mutex; - auto session = std::make_unique(client, db, nullptr, std::move(session_config)); + std::unique_ptr session{new Session{client, db, nullptr, std::move(session_config)}}; + + util::Mutex mutex; auto progress_handler = [&](uint_fast64_t downloaded_bytes, uint_fast64_t downloadable_bytes, uint_fast64_t uploaded_bytes, uint_fast64_t uploadable_bytes, @@ -3687,26 +3722,26 @@ TEST(Sync_UploadDownloadProgress_6) CHECK_EQUAL(uploadable_bytes, 0); CHECK_EQUAL(progress_version, 0); CHECK_EQUAL(snapshot_version, 1); - std::lock_guard lock{mutex}; + util::LockGuard lock{mutex}; session.reset(); }; session->set_progress_handler(progress_handler); { - std::lock_guard lock{mutex}; + util::LockGuard lock{mutex}; session->bind(); } - client.drain(); + client.stop(); server.stop(); + client_thread.join(); server_thread.join(); // The check is that we reach this point without deadlocking. } -// Event Loop TODO: re-enable test once errors are propogating -#if 0 + TEST(Sync_MultipleSyncAgentsNotAllowed) { // At most one sync agent is allowed to participate in a Realm file access @@ -3727,7 +3762,7 @@ TEST(Sync_MultipleSyncAgentsNotAllowed) session_2.bind("realm://foo/bar", "blablabla"); CHECK_THROW(client.run(), MultipleSyncAgents); } -#endif + TEST(Sync_CancelReconnectDelay) { @@ -4306,18 +4341,16 @@ TEST(Sync_MergeMultipleChangesets) TEST_DIR(dir); MultiClientServerFixture fixture(2, 1, dir, test_context); - - // Start server and upload changes of first client. Session session_1 = fixture.make_session(0, db_1); fixture.bind_session(session_1, 0, "/test"); Session session_2 = fixture.make_session(1, db_2); fixture.bind_session(session_2, 0, "/test"); + // Start server and upload changes of first client. fixture.start_server(0); fixture.start_client(0); session_1.wait_for_upload_complete_or_client_stopped(); session_1.wait_for_download_complete_or_client_stopped(); - session_1.detach(); // Stop first client. fixture.stop_client(0); @@ -4436,7 +4469,13 @@ TEST(Sync_ServerDiscardDeadConnections) BowlOfStonesSemaphore bowl; auto error_handler = [&](std::error_code ec, bool, const std::string&) { - CHECK_EQUAL(ec, sync::websocket::make_error_code(ErrorCodes::ReadError)); + using syserr = util::error::basic_system_errors; + bool valid_error = (util::MiscExtErrors::end_of_input == ec) || + (util::MiscExtErrors::premature_end_of_input == ec) || + // FIXME: this is the error on Windows. is it correct? + (util::make_basic_system_error_code(syserr::connection_reset) == ec) || + (util::make_basic_system_error_code(syserr::connection_aborted) == ec); + CHECK(valid_error); bowl.add_stone(); }; fixture.set_client_side_error_handler(std::move(error_handler)); @@ -5016,6 +5055,11 @@ TEST_IF(Sync_SSL_Certificates, false) client_config.reconnect_mode = ReconnectMode::testing; Client client(client_config); + ThreadWrapper client_thread; + client_thread.start([&] { + client.run(); + }); + Session::Config session_config; session_config.server_address = server_address[i]; session_config.server_port = 443; @@ -5043,6 +5087,8 @@ TEST_IF(Sync_SSL_Certificates, false) session.bind(); session.wait_for_download_complete_or_client_stopped(); + client.stop(); + client_thread.join(); } } @@ -6628,9 +6674,8 @@ TEST(Sync_NonIncreasingServerVersions) uint_fast64_t downloadable_bytes = 0; VersionInfo version_info; util::StderrLogger logger; - auto transact = db->start_read(); history.integrate_server_changesets(progress, &downloadable_bytes, server_changesets_encoded, version_info, - DownloadBatchState::SteadyState, logger, transact); + DownloadBatchState::SteadyState, logger); } TEST(Sync_InvalidChangesetFromServer) @@ -6656,9 +6701,8 @@ TEST(Sync_InvalidChangesetFromServer) VersionInfo version_info; util::StderrLogger logger; - auto transact = db->start_read(); CHECK_THROW_EX(history.integrate_server_changesets({}, nullptr, util::Span(&server_changeset, 1), version_info, - DownloadBatchState::SteadyState, logger, transact), + DownloadBatchState::LastInBatch, logger), sync::IntegrationException, StringData(e.what()).contains("Failed to parse received changeset: Invalid interned string")); } diff --git a/test/test_sync_history_migration.cpp b/test/test_sync_history_migration.cpp index bd3a196a85d..278c36511bd 100644 --- a/test/test_sync_history_migration.cpp +++ b/test/test_sync_history_migration.cpp @@ -321,7 +321,6 @@ TEST(Sync_HistoryMigration) auto get_server_path = [&](const std::string& server_dir) { fixtures::ClientServerFixture fixture{server_dir, test_context}; - fixture.start(); return fixture.map_virtual_to_real_path(virtual_path); }; diff --git a/test/test_util_error.cpp b/test/test_util_error.cpp index 7d9e0ec73d3..57137886956 100644 --- a/test/test_util_error.cpp +++ b/test/test_util_error.cpp @@ -68,9 +68,10 @@ TEST(BasicSystemErrors_Messages) { std::error_code err = make_error_code(error::address_family_not_supported); - CHECK_GREATER(err.message().length(), 0); -#ifndef _WIN32 // Older versions of the Windows CRT return "Unknown error" for this error instead of an actual message +#ifdef _WIN32 + CHECK_EQUAL(err.message(), error_message); +#else CHECK_NOT_EQUAL(err.message(), error_message); #endif } @@ -87,7 +88,9 @@ TEST(BasicSystemErrors_Messages) { std::error_code err = make_error_code(error::operation_aborted); CHECK_GREATER(err.message().length(), 0); -#ifndef _WIN32 // Older versions of the Windows CRT return "Unknown error" for this error instead of an actual message +#ifdef _WIN32 + CHECK_EQUAL(err.message(), error_message); +#else CHECK_NOT_EQUAL(err.message(), error_message); #endif } diff --git a/test/test_util_file.cpp b/test/test_util_file.cpp index ab2661b90c3..811d4bea60b 100644 --- a/test/test_util_file.cpp +++ b/test/test_util_file.cpp @@ -191,6 +191,21 @@ TEST(Utils_File_resolve) */ } +#ifndef _WIN32 // An open file cannot be deleted on Windows +TEST(Utils_File_remove_open) +{ + if (test_util::test_dir_is_exfat()) + return; + + std::string file_name = File::resolve("FooBar", test_util::get_test_path_prefix()); + File f(file_name, File::mode_Write); + + CHECK_EQUAL(f.is_removed(), false); + std::remove(file_name.c_str()); + CHECK_EQUAL(f.is_removed(), true); +} +#endif + TEST(Utils_File_RemoveDirRecursive) { TEST_DIR(dir_0); @@ -294,18 +309,4 @@ TEST(Utils_File_ForEach) } } -TEST(Utils_File_Lock) -{ - TEST_DIR(dir); - util::try_make_dir(dir); - auto file = File::resolve("test", dir); - File f1(file, File::mode_Write); - File f2(file); - CHECK(f1.try_rw_lock_exclusive()); - CHECK_NOT(f2.try_rw_lock_shared()); - f1.rw_unlock(); - CHECK(f1.try_rw_lock_shared()); - CHECK_NOT(f2.try_rw_lock_exclusive()); -} - #endif diff --git a/test/test_util_http.cpp b/test/test_util_http.cpp index 84aea574d55..74f5955a8a7 100644 --- a/test/test_util_http.cpp +++ b/test/test_util_http.cpp @@ -132,7 +132,7 @@ TEST(HTTP_RequestResponse) std::thread server_thread{[&] { BufferedSocket c(server); - HTTPServer http(c, test_context.logger); + HTTPServer http(c, *(test_context.logger)); acceptor.async_accept(c, [&](std::error_code ec) { CHECK(!ec); http.async_receive_request([&](HTTPRequest req, std::error_code ec) { @@ -156,7 +156,7 @@ TEST(HTTP_RequestResponse) { network::Service client; BufferedSocket c(client); - HTTPClient http(c, test_context.logger); + HTTPClient http(c, *(test_context.logger)); c.async_connect(ep, [&](std::error_code ec) { CHECK(!ec); HTTPRequest req; @@ -313,8 +313,8 @@ struct FakeHTTPParser : HTTPParserBase { StringData body; std::error_code error; - FakeHTTPParser(const std::shared_ptr& logger_ptr) - : HTTPParserBase{logger_ptr} + FakeHTTPParser(util::Logger& logger) + : HTTPParserBase{logger} { } @@ -339,7 +339,7 @@ struct FakeHTTPParser : HTTPParserBase { TEST(HTTPParser_ParseHeaderLine) { - FakeHTTPParser p{test_context.logger}; + FakeHTTPParser p{*(test_context.logger)}; struct expect { bool success; diff --git a/test/test_util_logger.cpp b/test/test_util_logger.cpp index 9522ecd78d3..20b23977b8c 100644 --- a/test/test_util_logger.cpp +++ b/test/test_util_logger.cpp @@ -86,100 +86,6 @@ TEST(Util_Logger_LevelToFromString) } -TEST(Util_Logger_LevelThreshold) -{ - using namespace realm::util; - auto base_logger = std::make_shared(); - auto threadsafe_logger = std::make_shared(base_logger); - auto prefix_logger = PrefixLogger("test", threadsafe_logger); // created using Logger shared_ptr - auto prefix_logger2 = PrefixLogger("test2", prefix_logger); // created using PrefixLogger - - CHECK(base_logger->get_level_threshold() == Logger::default_log_level); - CHECK(threadsafe_logger->get_level_threshold() == Logger::default_log_level); - CHECK(prefix_logger.get_level_threshold() == Logger::default_log_level); - CHECK(prefix_logger2.get_level_threshold() == Logger::default_log_level); - - base_logger->set_level_threshold(Logger::Level::error); - CHECK(base_logger->get_level_threshold() == Logger::Level::error); - CHECK(threadsafe_logger->get_level_threshold() == Logger::Level::error); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::error); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::error); - - threadsafe_logger->set_level_threshold(Logger::Level::trace); - CHECK(base_logger->get_level_threshold() == Logger::Level::trace); - CHECK(threadsafe_logger->get_level_threshold() == Logger::Level::trace); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::trace); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::trace); - - prefix_logger.set_level_threshold(Logger::Level::debug); - CHECK(base_logger->get_level_threshold() == Logger::Level::debug); - CHECK(threadsafe_logger->get_level_threshold() == Logger::Level::debug); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::debug); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::debug); - - prefix_logger2.set_level_threshold(Logger::Level::info); - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(threadsafe_logger->get_level_threshold() == Logger::Level::info); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::info); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::info); - - auto ll_logger = std::make_shared(base_logger); - auto ll_logger2 = std::make_shared(base_logger, Logger::Level::trace); - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(ll_logger->get_level_threshold() == Logger::Level::info); - CHECK(ll_logger2->get_level_threshold() == Logger::Level::trace); - - ll_logger->set_level_threshold(Logger::Level::error); - ll_logger2->set_level_threshold(Logger::Level::debug); - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(ll_logger->get_level_threshold() == Logger::Level::error); - CHECK(ll_logger2->get_level_threshold() == Logger::Level::debug); - - auto prefix_logger3 = PrefixLogger("test3", ll_logger); - auto prefix_logger4 = PrefixLogger("test4", ll_logger2); -} - - -TEST(Util_Logger_LocalThresholdLogger) -{ - using namespace realm::util; - auto base_logger = std::make_shared(); - auto lt_logger = std::make_shared(base_logger); - auto lt_logger2 = std::make_shared(base_logger, Logger::Level::trace); - auto prefix_logger = PrefixLogger("test", lt_logger); - auto prefix_logger2 = PrefixLogger("test2", lt_logger2); - - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(lt_logger->get_level_threshold() == Logger::Level::info); - CHECK(lt_logger2->get_level_threshold() == Logger::Level::trace); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::info); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::trace); - - lt_logger->set_level_threshold(Logger::Level::error); - lt_logger2->set_level_threshold(Logger::Level::debug); - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(lt_logger->get_level_threshold() == Logger::Level::error); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::error); - CHECK(lt_logger2->get_level_threshold() == Logger::Level::debug); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::debug); - - prefix_logger.set_level_threshold(Logger::Level::off); - prefix_logger2.set_level_threshold(Logger::Level::all); - CHECK(base_logger->get_level_threshold() == Logger::Level::info); - CHECK(lt_logger->get_level_threshold() == Logger::Level::off); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::off); - CHECK(lt_logger2->get_level_threshold() == Logger::Level::all); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::all); - - base_logger->set_level_threshold(Logger::Level::error); - CHECK(base_logger->get_level_threshold() == Logger::Level::error); - CHECK(lt_logger->get_level_threshold() == Logger::Level::off); - CHECK(prefix_logger.get_level_threshold() == Logger::Level::off); - CHECK(lt_logger2->get_level_threshold() == Logger::Level::all); - CHECK(prefix_logger2.get_level_threshold() == Logger::Level::all); -} - - TEST(Util_Logger_Stream) { std::ostringstream out_1; @@ -267,22 +173,18 @@ TEST(Util_Logger_File_2) } } + TEST(Util_Logger_Prefix) { std::ostringstream out_1; std::ostringstream out_2; { - auto root_logger = std::make_shared(out_1); - util::PrefixLogger logger1("Prefix: ", root_logger); - util::PrefixLogger logger2("Prefix2: ", logger1); - logger1.info("Foo"); + util::StreamLogger root_logger(out_1); + util::PrefixLogger logger("Prefix: ", root_logger); + logger.info("Foo"); out_2 << "Prefix: Foo\n"; - logger1.info("Bar"); + logger.info("Bar"); out_2 << "Prefix: Bar\n"; - logger2.info("Foo"); - out_2 << "Prefix: Prefix2: Foo\n"; - logger2.info("Bar"); - out_2 << "Prefix: Prefix2: Bar\n"; } CHECK(out_1.str() == out_2.str()); } diff --git a/test/test_util_network.cpp b/test/test_util_network.cpp index e9291d1c375..79dbb1dbd40 100644 --- a/test/test_util_network.cpp +++ b/test/test_util_network.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -153,69 +152,6 @@ TEST(Network_PostOperation) } -TEST(Network_RunUntilStopped) -{ - network::Service service; - auto post_to_service = [&](util::UniqueFunction func = {}) { - auto [promise, future] = util::make_promise_future(); - service.post([promise = std::move(promise), func = std::move(func)](Status s) mutable { - if (!s.is_ok()) { - promise.set_error(s); - return; - } - if (func) { - func(); - } - promise.emplace_value(); - }); - return std::move(future); - }; - - auto before_run = post_to_service(); - - auto [thread_stopped_promise, thread_stopped_future] = util::make_promise_future(); - std::thread thread([&service, promise = std::move(thread_stopped_promise)]() mutable { - service.run_until_stopped(); - promise.emplace_value(); - }); - - before_run.get(); - CHECK_NOT(thread_stopped_future.is_ready()); - // Post while it's running. This should get fulfilled immediately. - post_to_service().get(); - CHECK_NOT(thread_stopped_future.is_ready()); - - util::Optional> timer_ran_future; - network::DeadlineTimer timer(service); - post_to_service([&] { - auto [promise, future] = util::make_promise_future(); - timer.async_wait(std::chrono::milliseconds{250}, [promise = std::move(promise)](Status s) mutable { - if (!s.is_ok()) { - promise.set_error(s); - return; - } - promise.emplace_value(); - }); - timer_ran_future = std::move(future); - }).get(); - - CHECK_NOT(thread_stopped_future.is_ready()); - CHECK(timer_ran_future); - timer_ran_future->get(); - - service.stop(); - thread.join(); - - thread_stopped_future.get(); - - auto after_stop = post_to_service(); - CHECK_NOT(after_stop.is_ready()); - - service.reset(); - service.run(); - after_stop.get(); -} - TEST(Network_EventLoopStopAndReset_1) { network::Service service; diff --git a/test/test_util_websocket.cpp b/test/test_util_websocket.cpp index 7f2a5ad4e9d..3c361b45317 100644 --- a/test/test_util_websocket.cpp +++ b/test/test_util_websocket.cpp @@ -14,8 +14,8 @@ namespace { // A class for connecting two socket endpoints through a memory buffer. class Pipe { public: - Pipe(const std::shared_ptr& logger_ptr) - : m_logger_ptr(logger_ptr) + Pipe(util::Logger& logger) + : m_logger(logger) { } @@ -23,7 +23,7 @@ class Pipe { void async_write(const char* data, size_t size, WriteCompletionHandler handler) { - m_logger_ptr->trace("async_write, size = %1", size); + m_logger.trace("async_write, size = %1", size); m_buffer.insert(m_buffer.end(), data, data + size); do_read(); handler(std::error_code{}, size); @@ -31,7 +31,7 @@ class Pipe { void async_read(char* buffer, size_t size, ReadCompletionHandler handler) { - m_logger_ptr->trace("async_read, size = %1", size); + m_logger.trace("async_read, size = %1", size); REALM_ASSERT(!m_reader_waiting); m_reader_waiting = true; m_plain_async_read = true; @@ -43,7 +43,7 @@ class Pipe { void async_read_until(char* buffer, size_t size, char delim, ReadCompletionHandler handler) { - m_logger_ptr->trace("async_read_until, size = %1, delim = %2", size, delim); + m_logger.trace("async_read_until, size = %1, delim = %2", size, delim); REALM_ASSERT(!m_reader_waiting); m_reader_waiting = true; m_plain_async_read = false; @@ -56,7 +56,7 @@ class Pipe { private: - const std::shared_ptr m_logger_ptr; + util::Logger& m_logger; std::vector m_buffer; bool m_reader_waiting = false; @@ -71,8 +71,8 @@ class Pipe { void do_read() { - m_logger_ptr->trace("do_read(), m_buffer.size = %1, m_reader_waiting = %2, m_read_size = %3", m_buffer.size(), - m_reader_waiting, m_read_size); + m_logger.trace("do_read(), m_buffer.size = %1, m_reader_waiting = %2, m_read_size = %3", m_buffer.size(), + m_reader_waiting, m_read_size); if (!m_reader_waiting) return; @@ -92,7 +92,7 @@ class Pipe { void transfer(size_t size) { - m_logger_ptr->trace("transfer()"); + m_logger.trace("transfer()"); std::copy(m_buffer.begin(), m_buffer.begin() + size, m_read_buffer); m_buffer.erase(m_buffer.begin(), m_buffer.begin() + size); m_reader_waiting = false; @@ -101,7 +101,7 @@ class Pipe { void delim_not_found() { - m_logger_ptr->trace("delim_not_found"); + m_logger.trace("delim_not_found"); m_reader_waiting = false; m_handler(util::MiscExtErrors::delim_not_found, 0); } @@ -109,14 +109,16 @@ class Pipe { class PipeTest { public: + util::Logger& logger; Pipe pipe; std::string result; bool done = false; bool error = false; - PipeTest(const std::shared_ptr& logger_ptr) - : pipe(logger_ptr) + PipeTest(util::Logger& logger) + : logger(logger) + , pipe(logger) { } @@ -175,16 +177,16 @@ class WSConfig : public websocket::Config { std::vector ping_messages; std::vector pong_messages; - WSConfig(Pipe& pipe_in, Pipe& pipe_out, const std::shared_ptr& logger_ptr) + WSConfig(Pipe& pipe_in, Pipe& pipe_out, util::Logger& logger) : m_pipe_in(pipe_in) , m_pipe_out(pipe_out) - , m_logger_ptr(logger_ptr) + , m_logger(logger) { } - const std::shared_ptr& websocket_get_logger() noexcept override + util::Logger& websocket_get_logger() noexcept override { - return m_logger_ptr; + return m_logger; } std::mt19937_64& websocket_get_random() noexcept override @@ -265,22 +267,22 @@ class WSConfig : public websocket::Config { private: Pipe &m_pipe_in, &m_pipe_out; - const std::shared_ptr m_logger_ptr; + util::Logger& m_logger; std::mt19937_64 m_random; }; class Fixture { public: - const std::shared_ptr m_prefix_logger_1, m_prefix_logger_2, m_prefix_logger_3, m_prefix_logger_4; + util::PrefixLogger m_prefix_logger_1, m_prefix_logger_2, m_prefix_logger_3, m_prefix_logger_4; Pipe pipe_1, pipe_2; WSConfig config_1, config_2; websocket::Socket socket_1, socket_2; - Fixture(const std::shared_ptr& logger) - : m_prefix_logger_1{std::make_shared("Socket_1: ", logger)} - , m_prefix_logger_2{std::make_shared("Socket_2: ", logger)} - , m_prefix_logger_3{std::make_shared("Pipe_1: ", logger)} - , m_prefix_logger_4{std::make_shared("Pipe_2: ", logger)} + Fixture(util::Logger& logger) + : m_prefix_logger_1("Socket_1: ", logger) + , m_prefix_logger_2("Socket_2: ", logger) + , m_prefix_logger_3("Pipe_1: ", logger) + , m_prefix_logger_4("Pipe_2: ", logger) , pipe_1(m_prefix_logger_3) , pipe_2(m_prefix_logger_4) , config_1(pipe_1, pipe_2, m_prefix_logger_1) @@ -296,7 +298,7 @@ class Fixture { TEST(WebSocket_Pipe) { { - PipeTest pipe_test{test_context.logger}; + PipeTest pipe_test{*(test_context.logger)}; std::string input_1 = "Hello World"; pipe_test.write(input_1); pipe_test.read_plain(input_1.size()); @@ -343,7 +345,7 @@ TEST(WebSocket_Pipe) TEST(WebSocket_Messages) { - Fixture fixt{test_context.logger}; + Fixture fixt{*(test_context.logger)}; WSConfig& config_1 = fixt.config_1; WSConfig& config_2 = fixt.config_2; @@ -405,7 +407,7 @@ TEST(WebSocket_Messages) TEST(WebSocket_Fragmented_Messages) { - Fixture fixt{test_context.logger}; + Fixture fixt{*(test_context.logger)}; WSConfig& config_1 = fixt.config_1; WSConfig& config_2 = fixt.config_2; @@ -440,7 +442,7 @@ TEST(WebSocket_Fragmented_Messages) TEST(WebSocket_Interleaved_Fragmented_Messages) { - Fixture fixt{test_context.logger}; + Fixture fixt{*(test_context.logger)}; WSConfig& config_1 = fixt.config_1; WSConfig& config_2 = fixt.config_2; diff --git a/test/util/CMakeLists.txt b/test/util/CMakeLists.txt index 66c8b0cddb7..ccb7aaba926 100644 --- a/test/util/CMakeLists.txt +++ b/test/util/CMakeLists.txt @@ -30,6 +30,7 @@ set(TEST_UTIL_HEADERS resource_limits.hpp semaphore.hpp super_int.hpp + test_logger.hpp test_only.hpp test_path.hpp test_types.hpp diff --git a/test/util/test_logger.hpp b/test/util/test_logger.hpp new file mode 100644 index 00000000000..a8901d83622 --- /dev/null +++ b/test/util/test_logger.hpp @@ -0,0 +1,129 @@ +#ifndef REALM_TEST_UTIL_LOGGER_HPP +#define REALM_TEST_UTIL_LOGGER_HPP + +#include +#include +#include +#include + +namespace realm { +namespace test_util { + +/// A thread-safe Logger implementation that allows testing whether a particular +/// log message was emitted. +class TestLogger : public util::Logger, private util::Logger::LevelThreshold { +public: + using util::Logger::Level; + + /// Construct a TestLogger. If \a forward_to is non-null, a copy of each log + /// message will be forwarded to that logger. + TestLogger(util::Logger* forward_to = nullptr) + : util::Logger(static_cast(*this)) + , m_forward_to(forward_to) + { + if (forward_to == this) + forward_to = nullptr; + } + + /// Return true if a log message matching \a rx was emitted at the given log + /// level. If \a at_level is `Level::all`, log messages at all levels are + /// checked. + /// + /// The regular expression is matched using `std::regex_search()`, which + /// means that a substring match will return true (the regular expression is + /// not required to match the whole message). + /// + /// The level prefix ("INFO", "WARNING", etc.) is not part of the input to + /// the regular expression match. + /// + /// This method is thread-safe. + bool did_log(const std::regex& rx, Level at_level = Level::all, + std::regex_constants::match_flag_type = std::regex_constants::match_default) const; + + /// Return true if any message was emitted at (and only at) the given log + /// level. If \a at_level is `Level::all`, returns true if any log message + /// was emitted at any level. + /// + /// This method is thread-safe. + bool did_log(Level at_level) const; + + /// Write the whole log to \a os as if the log was emitted with the given + /// level. If \a at_level is `Level::all`, the full log is written out. + /// + /// This method is thread-safe. + void write(std::ostream& os, Level at_level = Level::all) const; + +protected: + /// Logger implementation. This method is thread-safe. + void do_log(Level, std::string message) override; + +private: + /// LevelThreshold implementation + Level get() const noexcept override final + { + return Level::all; + } + + struct Message { + Level level; + std::string message; + }; + + std::deque m_messages; + mutable std::mutex m_mutex; + util::Logger* m_forward_to; +}; + + +/// Implementation: + +inline bool TestLogger::did_log(const std::regex& rx, Level at_level, + std::regex_constants::match_flag_type flags) const +{ + std::lock_guard l(m_mutex); + for (const auto& message : m_messages) { + if (at_level == Level::all || message.level == at_level) { + if (std::regex_search(message.message, rx, flags)) + return true; + } + } + return false; +} + +inline bool TestLogger::did_log(Level at_level) const +{ + std::lock_guard l(m_mutex); + if (at_level == Level::all) + return m_messages.size() != 0; + + return std::any_of(begin(m_messages), end(m_messages), [&](auto& message) { + return message.level == at_level; + }); +} + +inline void TestLogger::write(std::ostream& os, Level threshold) const +{ + std::lock_guard l(m_mutex); + for (const auto& message : m_messages) { + if (message.level < threshold) + continue; + os << get_level_prefix(message.level); + os << message.message << '\n'; + } +} + +inline void TestLogger::do_log(Level level, std::string message) +{ + std::lock_guard l(m_mutex); + if (m_forward_to) { + m_forward_to->log(level, message.c_str()); + } + m_messages.emplace_back(Message{level, std::move(message)}); +} + + +} // namespace test_util +} // namespace realm + + +#endif // REALM_TEST_UTIL_LOGGER_HPP diff --git a/test/util/unit_test.cpp b/test/util/unit_test.cpp index 7ac67a1d2f2..408d0d7f780 100644 --- a/test/util/unit_test.cpp +++ b/test/util/unit_test.cpp @@ -409,6 +409,25 @@ class WildcardFilter : public Filter { patterns m_include, m_exclude; }; + +class IntraTestLogger : public Logger { +public: + IntraTestLogger(Logger& base_logger, Level threshold) + : util::Logger(threshold) + , m_base_logger(base_logger) + { + } + + void do_log(Logger::Level level, std::string const& message) override final + { + Logger::do_log(m_base_logger, level, message); // Throws + } + +private: + Logger& m_base_logger; +}; + + } // anonymous namespace @@ -436,10 +455,9 @@ class TestList::SharedContextImpl : public SharedContext { int num_ended_threads = 0; int last_thread_to_end = -1; - SharedContextImpl(const TestList& tests, int repetitions, int threads, - const std::shared_ptr& logger_ptr, Reporter& reporter, bool aof, - util::Logger::Level itll) - : SharedContext(tests, repetitions, threads, logger_ptr) + SharedContextImpl(const TestList& tests, int repetitions, int threads, util::Logger& logger, Reporter& reporter, + bool aof, util::Logger::Level itll) + : SharedContext(tests, repetitions, threads, logger) , reporter(reporter) , abort_on_failure(aof) , intra_test_log_level(itll) @@ -463,10 +481,9 @@ class TestList::ThreadContextImpl : public ThreadContext { std::atomic last_line_seen; bool errors_seen; - ThreadContextImpl(SharedContextImpl& sc, int ti, const std::shared_ptr& attached_logger) - : ThreadContext(sc, ti, attached_logger ? attached_logger : sc.report_logger_ptr) - , intra_test_logger( - std::make_shared(ThreadContext::report_logger_ptr, sc.intra_test_log_level)) + ThreadContextImpl(SharedContextImpl& sc, int ti, util::Logger* attached_logger) + : ThreadContext(sc, ti, attached_logger ? *attached_logger : sc.report_logger) + , intra_test_logger(std::make_shared(ThreadContext::report_logger, sc.intra_test_log_level)) , shared_context(sc) { } @@ -527,7 +544,7 @@ bool TestList::run(Config config) root_logger = std::make_shared(); // Throws } } - std::shared_ptr shared_logger = std::make_shared(root_logger); + util::ThreadSafeLogger shared_logger(root_logger); Reporter fallback_reporter; Reporter& reporter = config.reporter ? *config.reporter : fallback_reporter; @@ -589,7 +606,7 @@ bool TestList::run(Config config) config.abort_on_failure, config.intra_test_log_level); shared_context.concur_tests = std::move(concur_tests); shared_context.no_concur_tests = std::move(no_concur_tests); - std::vector> loggers(num_threads); + std::vector> loggers(num_threads); if (num_threads != 1 || !config.per_thread_log_path.empty()) { std::ostringstream formatter; formatter.imbue(std::locale::classic()); @@ -613,21 +630,21 @@ bool TestList::run(Config config) formatter.str(std::string()); formatter << a << std::setw(thread_digits) << (i + 1) << b; std::string path = formatter.str(); - shared_logger->info("Logging to %1", path); + shared_logger.info("Logging to %1", path); loggers[i].reset(new util::FileLogger(path)); } } } Timer timer; if (num_threads == 1) { - ThreadContextImpl thread_context(shared_context, 0, loggers[0]); + ThreadContextImpl thread_context(shared_context, 0, loggers[0].get()); thread_context.run(); thread_context.nonconcur_run(); } else { std::vector> thread_contexts(num_threads); for (int i = 0; i < num_threads; ++i) - thread_contexts[i].reset(new ThreadContextImpl(shared_context, i, loggers[i])); + thread_contexts[i].reset(new ThreadContextImpl(shared_context, i, loggers[i].get())); // First execute regular (concurrent) tests { diff --git a/test/util/unit_test.hpp b/test/util/unit_test.hpp index c272bf5a96b..a4e93a3abc7 100644 --- a/test/util/unit_test.hpp +++ b/test/util/unit_test.hpp @@ -578,14 +578,13 @@ class ThreadContext { /// The thread specific logger to be used by custom reporters. See also /// SharedContext::report_logger and TestContext::logger. - const std::shared_ptr report_logger_ptr; util::Logger& report_logger; ThreadContext(const ThreadContext&) = delete; ThreadContext& operator=(const ThreadContext&) = delete; protected: - ThreadContext(SharedContext&, int thread_index, const std::shared_ptr&); + ThreadContext(SharedContext&, int thread_index, util::Logger&); }; @@ -597,14 +596,13 @@ class SharedContext { /// The thread non-specific logger to be used by custom reporters. See also /// ThreadContext::report_logger. - const std::shared_ptr report_logger_ptr; util::Logger& report_logger; SharedContext(const SharedContext&) = delete; SharedContext& operator=(const SharedContext&) = delete; protected: - SharedContext(const TestList& tl, int nr, int nt, const std::shared_ptr& rl_ptr); + SharedContext(const TestList&, int num_recurrences, int num_threads, util::Logger&); }; @@ -771,25 +769,6 @@ void to_string(const T& value, std::string& str) str = out.str(); } -template -void to_string(const std::vector& value, std::string& str) -{ - std::ostringstream out; - SetPrecision::value>::exec(out); - - out << "{"; - bool first = true; - for (auto& v : value) { - if (!first) { - out << ", "; - } - out << v; - first = false; - } - out << "}"; - str = out.str(); -} - template void to_string(const std::optional& value, std::string& str) { @@ -932,20 +911,18 @@ inline bool TestContext::check_definitely_greater(long double a, long double b, return check_inexact_compare(cond, a, b, eps, file, line, "CHECK_DEFINITELY_GREATER", a_text, b_text, eps_text); } -inline ThreadContext::ThreadContext(SharedContext& sc, int ti, const std::shared_ptr& rl_ptr) +inline ThreadContext::ThreadContext(SharedContext& sc, int ti, util::Logger& rl) : shared_context(sc) , thread_index(ti) - , report_logger_ptr(rl_ptr) - , report_logger(*report_logger_ptr) + , report_logger(rl) { } -inline SharedContext::SharedContext(const TestList& tl, int nr, int nt, const std::shared_ptr& rl_ptr) +inline SharedContext::SharedContext(const TestList& tl, int nr, int nt, util::Logger& rl) : test_list(tl) , num_recurrences(nr) , num_threads(nt) - , report_logger_ptr(rl_ptr) - , report_logger(*report_logger_ptr) + , report_logger(rl) { } diff --git a/tools/cmake/SpecialtyBuilds.cmake b/tools/cmake/SpecialtyBuilds.cmake index 9603caa93dd..b37d6eb044c 100644 --- a/tools/cmake/SpecialtyBuilds.cmake +++ b/tools/cmake/SpecialtyBuilds.cmake @@ -56,7 +56,7 @@ option(REALM_LIBFUZZER "Compile with llvm libfuzzer" OFF) if(REALM_LIBFUZZER) if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") # todo: add the undefined sanitizer here after blacklisting false positives - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fsanitize=fuzzer,address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=fuzzer,address -fsanitize-coverage=trace-pc-guard") else() message(FATAL_ERROR "Compiling for libfuzzer is only supported with clang") From 12b35b8631799c1727c628efaa2ec4da8ebed338 Mon Sep 17 00:00:00 2001 From: Nicola Cabiddu Date: Thu, 16 Feb 2023 19:34:17 +0000 Subject: [PATCH 2/2] appease format checks --- src/realm/exec/realm_trawler.cpp | 14 +++++++++++--- src/realm/sync/network/default_socket.cpp | 4 ++-- src/realm/sync/noinst/client_impl_base.cpp | 6 +++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp index d1cc86308cc..df2cfce45c5 100644 --- a/src/realm/exec/realm_trawler.cpp +++ b/src/realm/exec/realm_trawler.cpp @@ -56,7 +56,9 @@ void consolidate_lists(std::vector& list, std::vector& list2) list.insert(list.end(), list2.begin(), list2.end()); list2.clear(); if (list.size() > 1) { - std::sort(begin(list), end(list), [](T& a, T& b) { return a.start < b.start; }); + std::sort(begin(list), end(list), [](T& a, T& b) { + return a.start < b.start; + }); auto prev = list.begin(); for (auto it = list.begin() + 1; it != list.end(); ++it) { @@ -79,7 +81,11 @@ void consolidate_lists(std::vector& list, std::vector& list2) } // Remove all of the now zero-size chunks from the free list - list.erase(std::remove_if(begin(list), end(list), [](T& chunk) { return chunk.length == 0; }), end(list)); + list.erase(std::remove_if(begin(list), end(list), + [](T& chunk) { + return chunk.length == 0; + }), + end(list)); } } @@ -505,7 +511,9 @@ std::string human_readable(uint64_t val) uint64_t get_size(const std::vector& list) { uint64_t sz = 0; - std::for_each(list.begin(), list.end(), [&](const Entry& e) { sz += e.length; }); + std::for_each(list.begin(), list.end(), [&](const Entry& e) { + sz += e.length; + }); return sz; } diff --git a/src/realm/sync/network/default_socket.cpp b/src/realm/sync/network/default_socket.cpp index f66aae84aaa..b6c80dbb26d 100644 --- a/src/realm/sync/network/default_socket.cpp +++ b/src/realm/sync/network/default_socket.cpp @@ -276,8 +276,8 @@ void DefaultWebSocketImpl::initiate_http_tunnel() if (response.status != HTTPStatus::Ok) { m_logger.error("Proxy server returned response '%1 %2'", response.status, response.reason); // Throws std::error_code ec2 = - websocket::Error::bad_response_unexpected_status_code; // FIXME: is this the right error? - m_observer.websocket_connect_error_handler(ec2); // Throws + websocket::Error::bad_response_unexpected_status_code; // FIXME: is this the right error? + m_observer.websocket_connect_error_handler(ec2); // Throws return; } diff --git a/src/realm/sync/noinst/client_impl_base.cpp b/src/realm/sync/noinst/client_impl_base.cpp index 6f27348a34d..ff76610be67 100644 --- a/src/realm/sync/noinst/client_impl_base.cpp +++ b/src/realm/sync/noinst/client_impl_base.cpp @@ -809,9 +809,9 @@ void Connection::initiate_ping_delay(milliseconds_type now) else if (!status.is_ok()) throw ExceptionForStatus(status); - handle_ping_delay(); // Throws - }); // Throws - logger.debug("Will emit a ping in %1 milliseconds", delay); // Throws + handle_ping_delay(); // Throws + }); // Throws + logger.debug("Will emit a ping in %1 milliseconds", delay); // Throws }